chore: reverse test
This commit is contained in:
parent
accd7c6f98
commit
bb89b90b1a
36
run_once.sh
36
run_once.sh
|
@ -37,27 +37,27 @@ python prepare_yaml_file.py ${run_type} ${model} ${max_steps} ${run_name} ${outp
|
||||||
|
|
||||||
export USE_MODELSCOPE_HUB=1
|
export USE_MODELSCOPE_HUB=1
|
||||||
|
|
||||||
# echo "Start recording npu status "
|
echo "Start recording npu status "
|
||||||
# bash npu_status.sh ${output_dir} 60 0 &
|
bash npu_status.sh ${output_dir} 60 0 &
|
||||||
# npu_status_pid=$!
|
npu_status_pid=$!
|
||||||
# # echo "${npu_status_pid}"
|
# echo "${npu_status_pid}"
|
||||||
|
|
||||||
# if [ "${gpu_cnt}" = "1" ]; then
|
if [ "${gpu_cnt}" = "1" ]; then
|
||||||
# ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
|
ASCEND_RT_VISIBLE_DEVICES=0 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
|
||||||
# train_pid=$!
|
train_pid=$!
|
||||||
# echo "Start single npu train"
|
echo "Start single npu train"
|
||||||
# else
|
else
|
||||||
# FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
|
FORCE_TORCHRUN=1 llamafactory-cli train ${output_dir}/${run_name}.yaml | tee "${output_dir}/log.txt" &
|
||||||
# train_pid=$!
|
train_pid=$!
|
||||||
# echo "Start multi npu train"
|
echo "Start multi npu train"
|
||||||
# fi
|
fi
|
||||||
|
|
||||||
# wait $train_pid
|
wait $train_pid
|
||||||
# echo "Train ended"
|
echo "Train ended"
|
||||||
|
|
||||||
# sleep 60
|
sleep 60
|
||||||
# kill $npu_status_pid
|
kill $npu_status_pid
|
||||||
# echo "Npu status ended"
|
echo "Npu status ended"
|
||||||
|
|
||||||
python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
|
python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
|
||||||
rm -r ${output_dir}
|
rm -r ${output_dir}
|
Loading…
Reference in New Issue