From 41c42f67a2183147f12dd0d1bb954cd43c636558 Mon Sep 17 00:00:00 2001 From: wql Date: Mon, 12 Aug 2024 16:51:45 +0800 Subject: [PATCH] change: change llama2_lora_sft.yaml --- examples/train_lora/llama2_lora_sft.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/train_lora/llama2_lora_sft.yaml b/examples/train_lora/llama2_lora_sft.yaml index 12f9d4d0..fc6c3554 100644 --- a/examples/train_lora/llama2_lora_sft.yaml +++ b/examples/train_lora/llama2_lora_sft.yaml @@ -5,31 +5,31 @@ model_name_or_path: modelscope/Llama-2-7b-ms stage: sft do_train: true finetuning_type: lora -lora_target: all +lora_target: q_proj,v_proj ### dataset dataset: alpaca_zh template: llama2 cutoff_len: 1024 -max_samples: 1000 +max_samples: 100000 overwrite_cache: true preprocessing_num_workers: 16 ### output -output_dir: ./saves/LLaMA2-7B/lora/train_24_8_12_15_46 -logging_steps: 10 -save_steps: 500 +output_dir: ./saves/LLaMA2-7B/lora/train_24_8_12_16_46 +logging_steps: 5 +save_steps: 100 plot_loss: true overwrite_output_dir: true ### train per_device_train_batch_size: 2 gradient_accumulation_steps: 8 -learning_rate: 1.0e-4 -num_train_epochs: 3.0 +learning_rate: 5e-5 +num_train_epochs: 5.0 lr_scheduler_type: cosine warmup_ratio: 0.1 -bf16: true +fp16: true ddp_timeout: 180000000 ### eval