From 87b6708a8b72ce508d94ab8328f7cde126f8a157 Mon Sep 17 00:00:00 2001
From: wql <wengqinlan18@163.com>
Date: Tue, 24 Sep 2024 09:32:12 +0800
Subject: [PATCH] feat: add fail mark for fail process

---
 run_once.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/run_once.sh b/run_once.sh
index 56a78b10..0f740e41 100644
--- a/run_once.sh
+++ b/run_once.sh
@@ -53,11 +53,16 @@ else
 fi
 
 wait $train_pid
+train_status=$?
 echo "Train ended"
 
 sleep 60
 kill $npu_status_pid
 echo "Npu status ended"
 
+if [ $train_status -ne 0 ]; then
+    output_dir="${output_dir}_fail"
+fi
+
 python3 -c "import moxing as mox; mox.file.copy_parallel('${output_dir}', 'obs://xty/results/${run_name}')"
 rm -r ${output_dir}
\ No newline at end of file