p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: reverse test

This commit is contained in:
wql 2024-09-19 16:57:15 +08:00
parent accd7c6f98
commit bb89b90b1a
1 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
run_once.sh
View File

@ -37,27 +37,27 @@ python prepare_yaml_file.py ${run_type} ${model} ${max_steps} ${run_name} ${outp
export USE_MODELSCOPE_HUB=1
# echo "Start recording npu status "
# bash npu_status.sh ${output_dir} 60 0 &
# npu_status_pid=$!
# # echo "${npu_status_pid}"
echo "Start recording npu status "
bash npu_status.sh ${output_dir} 60 0 &
npu_status_pid=$!
# echo "${npu_status_pid}"
# if [ "${gpu_cnt}" = "1" ]; then
# ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
# train_pid=$!
# echo "Start single npu train"
# else
# FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
# train_pid=$!
# echo "Start multi npu train"
# fi
if [ "${gpu_cnt}" = "1" ]; then
ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
train_pid=$!
echo "Start single npu train"
else
FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
train_pid=$!
echo "Start multi npu train"
fi
# wait $train_pid
# echo "Train ended"
wait $train_pid
echo "Train ended"
# sleep 60
# kill $npu_status_pid
# echo "Npu status ended"
sleep 60
kill $npu_status_pid
echo "Npu status ended"
python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
rm -r ${output_dir}