feat: add fail mark for fail process

run_once.sh

@@ -53,11 +53,16 @@ else
 fi
 
 wait $train_pid
+train_status=$?
 echo "Train ended"
 
 sleep 60
 kill $npu_status_pid
 echo "Npu status ended"
 
+if [ $train_status -ne 0 ]; then
+    output_dir="${output_dir}_fail"
+fi
+
 python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
 rm -r ${output_dir}