p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix examples

This commit is contained in:
hiyouga 2024-05-15 00:26:10 +08:00
parent 5bdad46387
commit 7e69e71a52
2 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
examples/deepspeed/ds_z0_config.json
View File

@ -14,5 +14,15 @@
}, },
"bf16": { "bf16": {
"enabled": "auto" "enabled": "auto"
},
"zero_optimization": {
"stage": 0,
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 5e8,
"contiguous_gradients": true,
"round_robin_gradients": true
} }
} }

2
examples/lora_multi_npu/ds_zero0.sh
View File

@ -12,4 +12,4 @@ ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 torchrun \
--node_rank $RANK \ --node_rank $RANK \
--master_addr $MASTER_ADDR \ --master_addr $MASTER_ADDR \
--master_port $MASTER_PORT \ --master_port $MASTER_PORT \
src/train.py examples/lora_multi_gpu/llama3_lora_sft_ds.yaml src/train.py examples/lora_multi_npu/llama3_lora_sft_ds.yaml