p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update examples

This commit is contained in:
hiyouga 2024-05-17 01:02:00 +08:00
parent 694a05fd04
commit ddec9e1b84
27 changed files with 155 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
examples/extras/badam/llama3_lora_sft.yaml
View File

@ -1,7 +1,7 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: full finetuning_type: full
@ -10,7 +10,7 @@ badam_switch_mode: descending
badam_switch_interval: 50 badam_switch_interval: 50
badam_verbose: 2 badam_verbose: 2
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -18,14 +18,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/full/sft output_dir: saves/llama3-8b/full/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -34,7 +34,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
pure_bf16: true pure_bf16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

14
examples/extras/fsdp_qlora/llama3_lora_sft.yaml
View File

@ -1,17 +1,17 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
quantization_bit: 4 quantization_bit: 4
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# ddp ### ddp
ddp_timeout: 180000000 ddp_timeout: 180000000
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -19,14 +19,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -35,7 +35,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/extras/galore/llama3_full_sft.yaml
View File

@ -1,7 +1,7 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: full finetuning_type: full
@ -11,7 +11,7 @@ galore_target: mlp,self_attn
galore_rank: 128 galore_rank: 128
galore_scale: 2.0 galore_scale: 2.0
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -19,14 +19,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/full/sft output_dir: saves/llama3-8b/full/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 1 gradient_accumulation_steps: 1
learning_rate: 0.0001 learning_rate: 0.0001
@ -35,7 +35,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
pure_bf16: true pure_bf16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/extras/llama_pro/llama3_freeze_sft.yaml
View File

@ -1,7 +1,7 @@
# model ### model
model_name_or_path: models/llama3-8b-instruct-pro model_name_or_path: models/llama3-8b-instruct-pro
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: freeze finetuning_type: freeze
@ -9,7 +9,7 @@ freeze_trainable_layers: 8
freeze_trainable_modules: all freeze_trainable_modules: all
use_llama_pro: true use_llama_pro: true
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -17,14 +17,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b-instruct-pro/freeze/sft output_dir: saves/llama3-8b-instruct-pro/freeze/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -33,7 +33,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/extras/loraplus/llama3_lora_sft.yaml
View File

@ -1,14 +1,14 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
loraplus_lr_ratio: 16.0 loraplus_lr_ratio: 16.0
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -16,14 +16,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/extras/mod/llama3_full_sft.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: full finetuning_type: full
mixture_of_depths: convert mixture_of_depths: convert
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b-mod/full/sft output_dir: saves/llama3-8b-mod/full/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
optim: paged_adamw_8bit optim: paged_adamw_8bit
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
pure_bf16: true pure_bf16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

10
examples/full_multi_gpu/llama3_full_predict.yaml
View File

@ -1,12 +1,12 @@
# model ### model
model_name_or_path: saves/llama3-8b/full/sft model_name_or_path: saves/llama3-8b/full/sft
# method ### method
stage: sft stage: sft
do_predict: true do_predict: true
finetuning_type: full finetuning_type: full
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -14,10 +14,10 @@ max_samples: 50
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/full/predict output_dir: saves/llama3-8b/full/predict
overwrite_output_dir: true overwrite_output_dir: true
# eval ### eval
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
predict_with_generate: true predict_with_generate: true

14
examples/full_multi_gpu/llama3_full_sft.yaml
View File

@ -1,16 +1,16 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: full finetuning_type: full
# ddp ### ddp
ddp_timeout: 180000000 ddp_timeout: 180000000
deepspeed: examples/deepspeed/ds_z3_config.json deepspeed: examples/deepspeed/ds_z3_config.json
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -18,14 +18,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/full/sft output_dir: saves/llama3-8b/full/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 2 gradient_accumulation_steps: 2
learning_rate: 0.0001 learning_rate: 0.0001
@ -34,7 +34,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

14
examples/lora_multi_gpu/llama3_lora_sft.yaml
View File

@ -1,16 +1,16 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# ddp ### ddp
ddp_timeout: 180000000 ddp_timeout: 180000000
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -18,14 +18,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 2 gradient_accumulation_steps: 2
learning_rate: 0.0001 learning_rate: 0.0001
@ -34,7 +34,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

14
examples/lora_multi_gpu/llama3_lora_sft_ds.yaml
View File

@ -1,17 +1,17 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# ddp ### ddp
ddp_timeout: 180000000 ddp_timeout: 180000000
deepspeed: examples/deepspeed/ds_z3_config.json deepspeed: examples/deepspeed/ds_z3_config.json
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -19,14 +19,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 2 gradient_accumulation_steps: 2
learning_rate: 0.0001 learning_rate: 0.0001
@ -35,7 +35,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

14
examples/lora_multi_npu/llama3_lora_sft_ds.yaml
View File

@ -1,17 +1,17 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# ddp ### ddp
ddp_timeout: 180000000 ddp_timeout: 180000000
deepspeed: examples/deepspeed/ds_z0_config.json deepspeed: examples/deepspeed/ds_z0_config.json
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -19,14 +19,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 2 gradient_accumulation_steps: 2
learning_rate: 0.0001 learning_rate: 0.0001
@ -35,7 +35,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/lora_single_gpu/llama3_lora_dpo.yaml
View File

@ -1,14 +1,14 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: dpo stage: dpo
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
dpo_ftx: 1.0 dpo_ftx: 1.0
# dataset ### dataset
dataset: orca_rlhf dataset: orca_rlhf
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -16,14 +16,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/dpo output_dir: saves/llama3-8b/lora/dpo
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.00001 learning_rate: 0.00001
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

10
examples/lora_single_gpu/llama3_lora_eval.yaml
View File

@ -1,19 +1,19 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
adapter_name_or_path: saves/llama3-8b/lora/sft adapter_name_or_path: saves/llama3-8b/lora/sft
# method ### method
finetuning_type: lora finetuning_type: lora
# dataset ### dataset
task: mmlu task: mmlu
split: test split: test
template: fewshot template: fewshot
lang: en lang: en
n_shot: 5 n_shot: 5
# output ### output
save_dir: saves/llama3-8b/lora/eval save_dir: saves/llama3-8b/lora/eval
# eval ### eval
batch_size: 4 batch_size: 4

12
examples/lora_single_gpu/llama3_lora_orpo.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: orpo stage: orpo
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: orca_rlhf dataset: orca_rlhf
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/orpo output_dir: saves/llama3-8b/lora/orpo
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.00001 learning_rate: 0.00001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/lora_single_gpu/llama3_lora_ppo.yaml
View File

@ -1,14 +1,14 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
reward_model: saves/llama3-8b/lora/reward reward_model: saves/llama3-8b/lora/reward
# method ### method
stage: ppo stage: ppo
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -16,14 +16,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/ppo output_dir: saves/llama3-8b/lora/ppo
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.00001 learning_rate: 0.00001
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# generate ### generate
max_new_tokens: 512 max_new_tokens: 512
top_k: 0 top_k: 0
top_p: 0.9 top_p: 0.9

10
examples/lora_single_gpu/llama3_lora_predict.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
adapter_name_or_path: saves/llama3-8b/lora/sft adapter_name_or_path: saves/llama3-8b/lora/sft
# method ### method
stage: sft stage: sft
do_predict: true do_predict: true
finetuning_type: lora finetuning_type: lora
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,10 +15,10 @@ max_samples: 50
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/predict output_dir: saves/llama3-8b/lora/predict
overwrite_output_dir: true overwrite_output_dir: true
# eval ### eval
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
predict_with_generate: true predict_with_generate: true

12
examples/lora_single_gpu/llama3_lora_pretrain.yaml
View File

@ -1,27 +1,27 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: pt stage: pt
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: c4_demo dataset: c4_demo
cutoff_len: 1024 cutoff_len: 1024
max_samples: 1000 max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -30,7 +30,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/lora_single_gpu/llama3_lora_reward.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: rm stage: rm
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: orca_rlhf dataset: orca_rlhf
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/reward output_dir: saves/llama3-8b/lora/reward
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.00001 learning_rate: 0.00001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/lora_single_gpu/llama3_lora_sft.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

8
examples/lora_single_gpu/llama3_preprocess.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -16,6 +16,6 @@ overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
tokenized_path: saves/llama3-8b/dataset/sft tokenized_path: saves/llama3-8b/dataset/sft
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
overwrite_output_dir: true overwrite_output_dir: true

12
examples/lora_single_gpu/llava1_5_lora_sft.yaml
View File

@ -1,14 +1,14 @@
# model ### model
model_name_or_path: llava-hf/llava-1.5-7b-hf model_name_or_path: llava-hf/llava-1.5-7b-hf
visual_inputs: true visual_inputs: true
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: mllm_demo dataset: mllm_demo
template: vicuna template: vicuna
cutoff_len: 1024 cutoff_len: 1024
@ -16,14 +16,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llava1_5-7b/lora/sft output_dir: saves/llava1_5-7b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

4
examples/merge_lora/llama3_gptq.yaml
View File

@ -1,8 +1,8 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
template: llama3 template: llama3
# export ### export
export_dir: models/llama3_gptq export_dir: models/llama3_gptq
export_quantization_bit: 4 export_quantization_bit: 4
export_quantization_dataset: data/c4_demo.json export_quantization_dataset: data/c4_demo.json

6
examples/merge_lora/llama3_lora_sft.yaml
View File

@ -1,12 +1,12 @@
# Note: DO NOT use quantized model or quantization_bit when merging lora adapters ### Note: DO NOT use quantized model or quantization_bit when merging lora adapters
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
adapter_name_or_path: saves/llama3-8b/lora/sft adapter_name_or_path: saves/llama3-8b/lora/sft
template: llama3 template: llama3
finetuning_type: lora finetuning_type: lora
# export ### export
export_dir: models/llama3_lora_sft export_dir: models/llama3_lora_sft
export_size: 2 export_size: 2
export_device: cpu export_device: cpu

12
examples/qlora_single_gpu/llama3_lora_sft_aqlm.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: ISTA-DASLab/Meta-Llama-3-8B-Instruct-AQLM-2Bit-1x16 model_name_or_path: ISTA-DASLab/Meta-Llama-3-8B-Instruct-AQLM-2Bit-1x16
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/qlora_single_gpu/llama3_lora_sft_awq.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-AWQ model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-AWQ
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/qlora_single_gpu/llama3_lora_sft_bitsandbytes.yaml
View File

@ -1,14 +1,14 @@
# model ### model
model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
quantization_bit: 4 quantization_bit: 4
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -16,14 +16,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -32,7 +32,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps

12
examples/qlora_single_gpu/llama3_lora_sft_gptq.yaml
View File

@ -1,13 +1,13 @@
# model ### model
model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ
# method ### method
stage: sft stage: sft
do_train: true do_train: true
finetuning_type: lora finetuning_type: lora
lora_target: q_proj,v_proj lora_target: q_proj,v_proj
# dataset ### dataset
dataset: identity,alpaca_gpt4_en dataset: identity,alpaca_gpt4_en
template: llama3 template: llama3
cutoff_len: 1024 cutoff_len: 1024
@ -15,14 +15,14 @@ max_samples: 1000
overwrite_cache: true overwrite_cache: true
preprocessing_num_workers: 16 preprocessing_num_workers: 16
# output ### output
output_dir: saves/llama3-8b/lora/sft output_dir: saves/llama3-8b/lora/sft
logging_steps: 10 logging_steps: 10
save_steps: 500 save_steps: 500
plot_loss: true plot_loss: true
overwrite_output_dir: true overwrite_output_dir: true
# train ### train
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 0.0001 learning_rate: 0.0001
@ -31,7 +31,7 @@ lr_scheduler_type: cosine
warmup_steps: 0.1 warmup_steps: 0.1
fp16: true fp16: true
# eval ### eval
val_size: 0.1 val_size: 0.1
per_device_eval_batch_size: 1 per_device_eval_batch_size: 1
evaluation_strategy: steps evaluation_strategy: steps