p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test: test moxing

This commit is contained in:
wql 2024-09-19 16:54:59 +08:00
parent 735b989587
commit accd7c6f98
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
run_once.sh
View File

@ -37,24 +37,27 @@ python prepare_yaml_file.py ${run_type} ${model} ${max_steps} ${run_name} ${outp
export USE_MODELSCOPE_HUB=1 export USE_MODELSCOPE_HUB=1
echo "Start recording npu status " # echo "Start recording npu status "
bash npu_status.sh ${output_dir} 60 0 & # bash npu_status.sh ${output_dir} 60 0 &
npu_status_pid=$! # npu_status_pid=$!
# echo "${npu_status_pid}" # # echo "${npu_status_pid}"
if [ "${gpu_cnt}" = "1" ]; then # if [ "${gpu_cnt}" = "1" ]; then
ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" & # ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
train_pid=$! # train_pid=$!
echo "Start single npu train" # echo "Start single npu train"
else # else
FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" & # FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
train_pid=$! # train_pid=$!
echo "Start multi npu train" # echo "Start multi npu train"
fi # fi
wait $train_pid # wait $train_pid
echo "Train ended" # echo "Train ended"
sleep 60 # sleep 60
kill $npu_status_pid # kill $npu_status_pid
echo "Npu status ended" # echo "Npu status ended"
python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
rm -r ${output_dir}