p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add fail mark for fail process

This commit is contained in:
wql 2024-09-24 09:32:12 +08:00
parent 4d27d046a0
commit 87b6708a8b
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
run_once.sh
View File

@ -53,11 +53,16 @@ else
fi fi
wait $train_pid wait $train_pid
train_status=$?
echo "Train ended" echo "Train ended"
sleep 60 sleep 60
kill $npu_status_pid kill $npu_status_pid
echo "Npu status ended" echo "Npu status ended"
if [ $train_status -ne 0 ]; then
output_dir="${output_dir}_fail"
fi
python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')" python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
rm -r ${output_dir} rm -r ${output_dir}