diff --git a/run_once.sh b/run_once.sh index 56a78b10..0f740e41 100644 --- a/run_once.sh +++ b/run_once.sh @@ -53,11 +53,16 @@ else fi wait $train_pid +train_status=$? echo "Train ended" sleep 60 kill $npu_status_pid echo "Npu status ended" +if [ $train_status -ne 0 ]; then + output_dir="${output_dir}_fail" +fi + python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')" rm -r ${output_dir} \ No newline at end of file