Merge branch 'main' of https://osredm.com/p04798526/LLaMA-Factory-310P3

results/inference_Baichuan2-7B_8_gpu_50_step_20240913031907/gpu_status_20240913031907.json

@@ -0,0 +1 @@
+{"cur_time": "2024-09-13 03:19:21", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 1692}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1177}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913031907/inference_Baichuan2-7B_8_gpu_50_step_20240913031907.yaml

@@ -0,0 +1,14 @@
+cutoff_len: 1024
+ddp_timeout: 180000000
+do_predict: true
+eval_dataset: alpaca_gpt4_zh
+include_tokens_per_second: true
+max_samples: 50
+model_name_or_path: ms_cache/hub/baichuan-inc/Baichuan2-7B-Base
+output_dir: ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913031907
+overwrite_cache: true
+overwrite_output_dir: true
+per_device_eval_batch_size: 2
+predict_with_generate: true
+preprocessing_num_workers: 16
+template: baichuan2

results/inference_Baichuan2-7B_8_gpu_50_step_20240913031907/log.txt

@@ -0,0 +1,2 @@
+09/13/2024 03:19:29 - INFO - llamafactory.hparams.parser - Process rank: 0, device: npu:0, n_gpu: 1, distributed training: False, compute dtype: None
+09/13/2024 03:19:30 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032104/gpu_status_20240913032104.json

@@ -0,0 +1 @@
+{"cur_time": "2024-09-13 03:21:17", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 1692}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1177}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032104/inference_Baichuan2-7B_8_gpu_50_step_20240913032104.yaml

@@ -0,0 +1,14 @@
+cutoff_len: 1024
+ddp_timeout: 180000000
+do_predict: true
+eval_dataset: alpaca_gpt4_zh
+include_tokens_per_second: true
+max_samples: 50
+model_name_or_path: ms_cache/hub/baichuan-inc/Baichuan2-7B-Base
+output_dir: ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913032104
+overwrite_cache: true
+overwrite_output_dir: true
+per_device_eval_batch_size: 2
+predict_with_generate: true
+preprocessing_num_workers: 16
+template: baichuan2

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032104/log.txt

@@ -0,0 +1,2 @@
+09/13/2024 03:21:25 - INFO - llamafactory.hparams.parser - Process rank: 0, device: npu:0, n_gpu: 1, distributed training: False, compute dtype: None
+09/13/2024 03:21:26 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032231/gpu_status_20240913032231.json

@@ -0,0 +1 @@
+{"cur_time": "2024-09-13 03:22:45", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 1692}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1177}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032231/inference_Baichuan2-7B_8_gpu_50_step_20240913032231.yaml

@@ -0,0 +1,14 @@
+cutoff_len: 1024
+ddp_timeout: 180000000
+do_predict: true
+eval_dataset: alpaca_gpt4_zh
+include_tokens_per_second: true
+max_samples: 50
+model_name_or_path: ms_cache/hub/baichuan-inc/Baichuan2-7B-Base
+output_dir: ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913032231
+overwrite_cache: true
+overwrite_output_dir: true
+per_device_eval_batch_size: 2
+predict_with_generate: true
+preprocessing_num_workers: 16
+template: baichuan2

results/inference_Baichuan2-7B_8_gpu_50_step_20240913032231/log.txt

@@ -0,0 +1,2 @@
+09/13/2024 03:22:53 - INFO - llamafactory.hparams.parser - Process rank: 0, device: npu:0, n_gpu: 1, distributed training: False, compute dtype: None
+09/13/2024 03:22:54 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...

results/inference_Baichuan2-7B_8_gpu_50_step_20240913053642/gpu_status_20240913053642.json

@@ -0,0 +1,13 @@
+{"cur_time": "2024-09-13 05:36:56", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 1692}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1177}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:38:08", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 7875
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:39:21", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 18803/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:40:34", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 18803/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:41:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 19787/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:42:59", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 19787/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 05:44:12", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20771/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913053642/inference_Baichuan2-7B_8_gpu_50_step_20240913053642.yaml

@@ -0,0 +1,14 @@
+cutoff_len: 1024
+ddp_timeout: 180000000
+do_predict: true
+eval_dataset: alpaca_gpt4_zh
+include_tokens_per_second: true
+max_samples: 50
+model_name_or_path: ms_cache/hub/baichuan-inc/Baichuan2-7B-Base
+output_dir: ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913053642
+overwrite_cache: true
+overwrite_output_dir: true
+per_device_eval_batch_size: 2
+predict_with_generate: true
+preprocessing_num_workers: 16
+template: baichuan2

results/inference_Baichuan2-7B_8_gpu_50_step_20240913053642/log.txt

@@ -0,0 +1,10 @@
+09/13/2024 05:37:03 - INFO - llamafactory.hparams.parser - Process rank: 0, device: npu:0, n_gpu: 1, distributed training: False, compute dtype: None
+09/13/2024 05:37:04 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
+eval example:
+input_ids:
+[195, 2848, 11450, 4755, 4434, 66, 196]
+inputs:
+<reserved_106>保持健康的三个提示。<reserved_107>
+09/13/2024 05:37:24 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
+09/13/2024 05:38:52 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
+09/13/2024 05:38:52 - INFO - llamafactory.model.loader - all params: 7,505,973,248

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/all_results.json

@@ -0,0 +1,10 @@
+{
+    "predict_bleu-4": 1.7100479999999998,
+    "predict_model_preparation_time": 0.0123,
+    "predict_rouge-1": 13.384794,
+    "predict_rouge-2": 2.707682,
+    "predict_rouge-l": 6.751074,
+    "predict_runtime": 6716.3545,
+    "predict_samples_per_second": 0.007,
+    "predict_steps_per_second": 0.004
+}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/gpu_status_20240913061013.json

@@ -0,0 +1,187 @@
+{"cur_time": "2024-09-13 06:10:27", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 1692}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1177}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:11:41", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 8799
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:12:54", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 18802/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:14:07", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 18803/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:15:21", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 19787/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:16:34", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 19787/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:17:47", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20771/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:19:01", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:20:14", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:21:27", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:22:41", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:23:54", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 06:25:07", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1384}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:26:21", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:27:34", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:28:47", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:30:00", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:31:14", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:32:27", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:33:41", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:34:54", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:36:07", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:37:21", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:38:34", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:39:47", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:41:00", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:42:14", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1483}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1383}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:43:27", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:44:40", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:45:54", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20782/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 06:47:07", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20783/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:48:20", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:49:34", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:50:47", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:52:00", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:53:14", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:54:27", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:55:40", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:56:54", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20784/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:58:07", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 06:59:20", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20784/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:00:33", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:01:47", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 07:03:00", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:04:13", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:05:26", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 07:06:40", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:07:53", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:09:06", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:10:20", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:11:33", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:12:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:13:59", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1484}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1382}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:15:13", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:16:26", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:17:39", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:18:53", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:20:06", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:21:19", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:22:33", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:23:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 07:24:59", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:26:13", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:27:26", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:28:39", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:29:53", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:31:06", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:32:19", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:33:33", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:34:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:35:59", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:37:13", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:38:26", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:39:39", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:40:53", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:42:06", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20784/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1342}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:43:19", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:44:32", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:45:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:46:59", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 07:48:13", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:49:26", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:50:39", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:51:53", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:53:06", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:54:19", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:55:33", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1176}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:56:46", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1528}]}
+{"cur_time": "2024-09-13 07:57:58", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1485}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 07:59:11", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 08:00:24", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20785/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1381}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 08:01:37", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1175}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1819}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}
+{"cur_time": "2024-09-13 08:02:49", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1050}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1341}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1526}]}
+{"cur_time": "2024-09-13 08:04:02", "npu_power_dissipation": [{"npu_id": 2, "power_dissipation": }, {"npu_id": 3, "power_dissipation": }, {"npu_id": 5, "power_dissipation": }, {"npu_id": 6, "power_dissipation": }], "device_mem_usage": [{"npu_id": 2, "chip_id": 0, "device_id": 0, "mem_usage": 20786/
+|}, {"npu_id": 2, "chip_id": 1, "device_id": 1, "mem_usage": 1174}, {"npu_id": 3, "chip_id": 0, "device_id": 2, "mem_usage": 1820}, {"npu_id": 3, "chip_id": 1, "device_id": 3, "mem_usage": 1049}, {"npu_id": 5, "chip_id": 0, "device_id": 4, "mem_usage": 1486}, {"npu_id": 5, "chip_id": 1, "device_id": 5, "mem_usage": 1380}, {"npu_id": 6, "chip_id": 0, "device_id": 6, "mem_usage": 1340}, {"npu_id": 6, "chip_id": 1, "device_id": 7, "mem_usage": 1527}]}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/inference_Baichuan2-7B_8_gpu_50_step_20240913061013.yaml

@@ -0,0 +1,14 @@
+cutoff_len: 1024
+ddp_timeout: 180000000
+do_predict: true
+eval_dataset: alpaca_gpt4_zh
+include_tokens_per_second: true
+max_samples: 50
+model_name_or_path: ms_cache/hub/baichuan-inc/Baichuan2-7B-Base
+output_dir: ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013
+overwrite_cache: true
+overwrite_output_dir: true
+per_device_eval_batch_size: 2
+predict_with_generate: true
+preprocessing_num_workers: 16
+template: baichuan2

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/log.txt

@@ -0,0 +1,20 @@
+09/13/2024 06:10:36 - INFO - llamafactory.hparams.parser - Process rank: 0, device: npu:0, n_gpu: 1, distributed training: False, compute dtype: None
+09/13/2024 06:10:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
+eval example:
+input_ids:
+[195, 2848, 11450, 4755, 4434, 66, 196]
+inputs:
+<reserved_106>保持健康的三个提示。<reserved_107>
+09/13/2024 06:10:46 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
+09/13/2024 06:12:15 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
+09/13/2024 06:12:15 - INFO - llamafactory.model.loader - all params: 7,505,973,248
+***** predict metrics *****
+  predict_bleu-4                 =       1.71
+  predict_model_preparation_time =     0.0123
+  predict_rouge-1                =    13.3848
+  predict_rouge-2                =     2.7077
+  predict_rouge-l                =     6.7511
+  predict_runtime                = 1:51:56.35
+  predict_samples_per_second     =      0.007
+  predict_steps_per_second       =      0.004
+09/13/2024 08:04:11 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/generated_predictions.jsonl

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/predict_results.json

@@ -0,0 +1,10 @@
+{
+    "predict_bleu-4": 1.7100479999999998,
+    "predict_model_preparation_time": 0.0123,
+    "predict_rouge-1": 13.384794,
+    "predict_rouge-2": 2.707682,
+    "predict_rouge-l": 6.751074,
+    "predict_runtime": 6716.3545,
+    "predict_samples_per_second": 0.007,
+    "predict_steps_per_second": 0.004
+}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/trainer_log.jsonl

@@ -0,0 +1,5 @@
+{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:14:07", "remaining_time": "0:56:29"}
+{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:34:52", "remaining_time": "0:52:19"}
+{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:55:04", "remaining_time": "0:36:43"}
+{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "1:23:03", "remaining_time": "0:20:45"}
+{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "1:46:10", "remaining_time": "0:00:00"}

results/inference_Baichuan2-7B_8_gpu_50_step_20240913061013/trainer_state.json

@@ -0,0 +1,32 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": null,
+  "eval_steps": 500,
+  "global_step": 0,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [],
+  "logging_steps": 500,
+  "max_steps": 0,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 0,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0,
+  "train_batch_size": null,
+  "trial_name": null,
+  "trial_params": null
+}