p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity
LLaMA-Factory-Mirror/results/gpu_status/gpu_status_0825_2.json

455 lines
512 KiB
JSON
Raw Blame History

{"cur_time": "2024-08-25 15:32:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.329}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 312.331}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 287.265}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 289.709}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 293.466}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 303.805}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 300.55}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:33:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.391}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 364.32}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 337.886}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 337.266}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 345.633}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 330.306}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 333.506}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:34:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.26}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.586}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.46}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.617}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.997}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.561}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 484.011}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:35:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.088}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.683}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.746}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.517}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.337}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.482}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.066}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:36:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.239}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.502}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.98}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.415}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.304}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.868}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:37:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.4}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 450.029}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 446.028}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 421.058}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 434.632}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 428.884}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 420.085}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:38:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.242}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 243.199}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 209.511}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 236.659}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 209.393}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.386}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 203.86}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:39:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.072}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 232.809}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 239.371}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 239.3}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 214.295}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 240.724}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 229.722}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:40:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.014}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 492.682}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.462}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.334}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.693}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.347}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.066}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:41:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.998}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.602}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 458.435}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 450.122}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.714}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 458.431}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:42:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.132}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.534}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.929}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 490.503}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.504}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.021}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.008}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:43:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.651}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.409}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.542}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.065}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.521}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.155}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:44:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.119}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.149}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.961}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.216}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.664}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.685}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.662}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:45:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.13}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 476.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.36}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.566}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.088}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 465.264}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:46:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.935}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.268}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.529}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.356}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.004}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.699}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:47:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.132}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.217}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.642}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.194}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.606}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.553}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.353}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:48:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.254}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.894}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.033}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.83}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.166}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.402}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:49:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.415}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.744}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.307}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.587}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.898}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.626}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 467.899}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:50:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.401}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 385.264}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 387.73}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 393.36}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 389.611}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 401.071}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 404.613}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:51:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.406}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 303.535}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 290.144}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 302.232}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 288.744}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 313.815}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 284.863}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:52:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.065}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 305.768}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 302.161}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 262.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 262.002}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 282.342}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 276.762}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:53:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.123}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 494.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.563}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 489.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 486.374}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.797}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 489.159}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:54:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.348}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.421}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.073}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.827}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.136}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.643}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.136}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:55:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.3}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.898}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.897}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.97}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 478.14}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:56:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.076}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.585}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.457}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.151}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.022}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.356}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:57:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.03}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.4}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.889}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.972}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.181}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.443}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.708}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:58:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.076}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.258}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.065}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.692}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.771}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.603}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.818}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 15:59:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.081}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.899}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.83}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.867}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.96}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.366}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:00:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.154}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.551}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.67}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.147}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.229}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.8}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.792}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:01:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.138}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.204}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.678}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.758}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.304}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.157}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.56}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:02:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.036}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.527}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.772}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.807}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.121}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.884}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.756}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:03:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.306}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 420.933}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 409.841}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 406.795}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 402.774}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 430.985}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 415.54}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:04:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.068}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 349.909}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 343.539}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 340.727}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 332.04}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 353.051}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 328.545}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:05:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.161}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 284.078}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 275.647}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 262.365}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 280.261}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 280.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 249.777}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:06:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.981}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 456.576}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 431.574}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 414.068}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 432.144}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 452.777}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 451.463}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:07:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.986}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.364}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.196}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.523}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.952}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.175}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:08:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.301}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.844}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.657}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.191}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.498}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.773}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:09:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.21}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.148}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.726}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.932}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:10:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.055}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.199}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.114}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.954}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.972}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.127}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.782}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:11:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.115}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.771}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.653}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.863}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.817}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.417}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.819}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:12:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.167}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.788}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.173}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.855}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.124}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:13:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.279}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.889}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.112}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.12}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.932}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.716}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.908}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:14:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.263}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 449.83}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 432.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 428.645}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 435.339}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 421.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 415.387}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:15:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.09}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 435.625}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 407.747}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 422.424}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 406.585}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 429.651}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 421.967}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:16:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.3}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 248.103}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 248.521}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 278.033}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 259.809}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 258.977}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 244.889}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:17:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.266}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 271.714}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 260.191}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 271.989}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 241.036}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 266.733}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 269.866}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:18:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.111}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 254.834}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 284.526}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 276.534}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 235.368}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 265.54}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 241.787}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:19:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.172}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 431.487}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 419.862}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 406.181}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 401.888}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 402.741}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 408.323}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:20:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.139}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 473.696}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 450.097}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 462.179}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 443.898}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 458.596}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 458.028}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:21:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.045}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.198}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.844}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 492.948}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.825}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.01}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 489.784}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:22:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.111}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.722}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.291}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.964}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.093}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.835}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.413}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:23:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.673}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.609}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.506}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.208}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.923}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.394}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:24:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.131}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.86}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.517}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.207}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.103}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:25:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.384}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.813}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.842}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.948}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.534}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.822}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:26:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.196}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 476.374}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.406}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.807}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.928}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.228}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.038}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:27:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.402}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 375.608}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 361.074}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 384.267}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 349.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 387.183}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 386.454}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:28:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 251.702}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 244.144}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 245.678}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 234.821}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 248.794}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 241.686}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:29:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.186}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 237.752}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 233.399}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 231.394}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 228.95}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 231.055}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 230.123}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:30:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.195}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.644}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 451.277}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 452.242}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.562}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 450.279}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 464.736}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:31:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.244}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 419.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 435.478}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 408.679}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 413.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 427.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 419.567}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:32:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.297}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.445}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.32}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.745}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.544}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.88}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.001}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:33:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.022}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.148}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.694}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.885}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.657}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.7}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.757}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:34:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.325}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.078}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 442.913}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 447.532}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 457.129}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.75}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 451.42}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:35:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.243}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 350.473}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 357.431}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 334.127}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 350.363}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 359.289}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 338.351}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:36:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.237}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 250.651}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.774}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 214.041}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 230.599}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 224.652}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:37:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.082}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.01}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.617}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 214.25}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 215.189}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 210.544}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:38:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.29}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 348.045}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 326.935}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 338.165}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 334.618}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 326.842}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 331.807}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:39:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.027}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 390.147}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 411.117}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 419.161}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 407.134}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 396.938}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 401.049}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:40:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 472.816}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 492.686}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 489.174}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 485.324}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.236}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.771}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:41:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.134}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.95}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.937}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.451}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.933}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.461}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:42:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.207}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.073}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.684}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.505}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.062}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.85}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.153}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:43:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.345}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.378}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.421}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.147}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.093}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.89}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.085}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:44:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.235}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 428.043}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 407.088}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 417.085}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 415.497}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 446.288}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 398.539}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:45:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.13}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 284.133}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 262.68}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.587}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 255.396}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 266.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 240.591}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:46:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.043}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 152.492}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 147.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 145.715}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 142.484}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 146.895}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 142.519}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:47:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.222}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.789}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 217.265}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.901}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 218.317}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 217.737}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 215.321}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:48:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.991}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 353.229}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 376.765}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 367.673}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 376.056}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.503}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 376.454}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:49:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.93}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.77}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.856}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 484.186}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.646}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.237}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:50:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.875}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.266}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.332}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.967}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.992}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.172}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:51:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.109}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.608}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.282}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.999}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.608}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.605}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.84}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:52:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.205}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.953}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.101}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.969}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.477}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.214}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.767}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:53:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.102}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 467.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.083}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.715}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.09}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.816}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.062}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:54:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.093}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 313.506}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 343.206}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 320.8}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 301.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 314.442}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 319.36}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:55:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.175}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 247.617}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 253.304}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 240.894}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 242.994}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 249.664}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 242.812}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:56:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.063}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 339.223}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 314.829}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 332.57}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 305.242}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 310.675}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 303.993}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:57:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.077}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.029}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.991}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.543}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.833}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.946}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 478.133}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:58:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.232}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.889}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.785}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.274}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.645}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.82}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.517}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 16:59:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.069}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 475.135}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.7}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.108}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.294}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.617}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 466.601}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:00:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.998}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.252}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.646}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.905}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.199}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.421}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.388}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:01:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.164}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.786}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.807}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.254}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.367}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.876}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.441}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:02:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.005}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 391.216}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 420.413}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 392.187}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 391.293}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 390.765}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 382.501}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:03:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.174}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 312.517}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 290.322}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 304.518}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 295.172}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 282.236}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 272.536}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:04:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.135}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.203}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.05}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.838}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 212.301}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.142}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 213.493}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:05:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.955}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 218.353}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.088}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 207.904}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 206.386}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 209.661}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 207.98}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:06:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.001}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 383.122}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 405.178}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 390.358}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 395.787}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 378.649}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 393.267}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:07:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.129}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 490.612}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 492.273}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.821}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.193}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 456.292}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.575}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:08:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.04}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.614}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.877}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.112}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.768}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.563}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:09:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.526}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.643}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.608}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.531}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.7}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.745}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:10:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.177}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.127}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.243}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.975}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.935}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.247}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.972}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:11:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.042}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.373}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.326}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.245}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.45}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.738}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.354}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:12:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.093}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.148}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.759}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.322}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.399}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.1}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.784}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:13:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.114}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.204}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.902}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.931}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.751}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.442}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.605}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:14:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.114}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.928}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.566}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.862}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.299}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.273}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.915}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:15:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.139}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 460.908}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 451.246}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 459.633}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 453.985}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.527}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 451.227}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:16:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.981}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 393.392}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 395.461}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 379.889}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 362.946}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 385.453}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 368.844}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:17:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.017}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 358.668}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 342.202}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 326.685}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 348.427}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 347.256}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 346.873}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:18:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.182}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 263.47}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 251.684}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 247.616}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 251.574}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 251.838}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 260.063}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:19:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.051}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 406.262}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 399.987}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 418.381}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 392.888}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 386.49}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 391.578}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:20:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.172}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.106}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.549}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.951}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.718}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.669}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:21:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.004}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.988}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.639}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.533}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.317}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.372}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.459}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:22:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.378}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.688}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.013}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.396}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.881}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.323}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:23:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.179}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 447.818}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 462.565}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 458.645}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 452.871}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 458.874}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 442.838}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:24:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.026}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 375.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.461}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 366.175}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 350.338}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 387.66}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 352.784}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:25:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.035}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 218.201}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 211.748}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 213.535}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 205.823}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 214.199}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 207.262}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:26:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.035}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 226.686}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 225.105}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 225.05}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 215.608}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.635}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 220.234}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:27:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.98}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 433.465}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 436.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 417.225}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 431.08}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 445.272}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 445.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:28:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.034}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.745}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.642}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.08}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.157}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.675}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.741}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:29:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.015}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.833}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.302}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.541}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.678}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.243}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.969}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:30:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.922}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.137}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.598}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.595}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.416}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.418}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.35}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:31:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.161}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.608}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.45}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.669}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.936}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.059}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:32:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.169}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.675}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.808}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.664}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.815}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.486}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.632}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:33:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.029}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.413}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.269}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.363}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.091}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.373}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.831}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:34:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.976}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.724}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.836}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.336}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.452}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.305}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.594}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:35:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.051}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 245.432}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 242.034}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 241.019}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 233.525}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 243.108}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 245.226}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:36:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.974}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 330.841}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 329.408}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 340.996}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 337.291}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 318.223}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 327.649}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:37:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.07}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 464.854}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 461.771}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 448.335}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 463.501}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.033}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 445.815}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:38:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.127}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.791}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.286}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.307}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.561}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.941}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.639}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:39:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.05}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.371}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.604}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.87}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.216}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.739}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:40:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.839}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.456}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.783}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.038}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.838}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.777}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.124}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:41:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.866}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.419}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.89}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.114}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.451}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.327}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.848}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:42:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.954}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.401}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.016}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.358}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.58}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.677}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:43:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.017}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 370.633}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.794}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 362.804}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 350.576}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 360.211}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 344.198}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:44:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.114}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 272.521}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 283.571}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.276}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 270.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 266.689}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 281.659}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:45:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.143}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.345}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.546}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.035}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 242.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 235.165}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 226.148}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:46:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.009}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 441.053}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 408.464}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 414.912}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 433.753}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 427.315}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 407.916}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:47:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.025}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 445.172}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 416.735}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 408.342}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 414.194}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.641}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 404.657}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:48:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 494.821}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.92}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.057}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 488.124}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 489.778}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 487.431}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:49:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.772}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.33}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.147}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.0}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.782}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.958}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:50:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.961}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.576}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.358}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.97}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.506}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.718}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.603}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:51:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.064}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.696}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.745}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.853}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.902}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.157}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.636}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:52:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.015}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 313.362}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 300.508}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 319.169}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 322.243}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 299.896}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 311.087}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:53:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.971}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 419.855}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 416.317}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 415.885}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 397.01}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 418.825}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 421.018}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:54:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.02}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 367.831}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 367.316}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 365.939}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 357.989}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 366.5}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 360.884}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:55:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.085}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 320.069}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 333.891}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 328.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 327.513}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 329.697}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 339.995}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:56:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.895}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 491.02}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.243}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.737}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.337}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 490.398}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:57:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.727}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.458}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.641}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.129}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.456}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.355}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.775}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:58:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.905}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.788}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.361}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.891}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.177}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.299}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.542}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 17:59:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.036}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.388}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.914}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.581}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.416}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.01}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.363}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:00:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.969}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 362.479}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 374.03}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 351.681}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 355.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 351.895}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:01:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.106}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 227.005}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.489}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 220.079}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 214.21}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.426}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 220.747}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:02:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.054}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 283.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 271.798}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 264.612}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 257.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 269.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 283.814}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:03:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.87}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 365.783}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 359.965}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.57}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 357.934}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 373.519}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 363.739}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:04:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.03}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.734}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.718}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 484.45}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.216}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 491.308}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:05:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.072}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.996}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.055}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.022}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.112}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.62}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:06:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.864}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.212}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.919}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.611}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.182}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.407}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.865}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:07:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.995}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.545}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.918}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.457}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.285}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.541}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.38}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:08:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.979}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.131}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.124}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 461.982}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 463.132}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.562}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.233}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:09:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.059}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 376.316}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.46}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 374.856}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 357.161}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 361.394}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 366.025}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:10:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.93}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 353.245}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 328.308}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 323.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 319.007}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 331.37}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 348.799}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:11:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.939}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 378.084}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.67}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.612}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 369.376}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 358.958}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 385.393}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:12:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.102}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 228.818}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 231.512}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 217.157}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 224.369}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.508}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 218.505}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:13:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.068}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 239.047}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 229.111}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.315}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 213.147}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 217.51}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 226.639}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:14:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 355.376}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 379.589}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 359.216}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 352.354}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.796}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 364.171}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:15:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.874}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 469.725}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 462.61}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.113}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 432.508}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 453.743}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 443.282}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:16:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.96}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.403}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.342}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.243}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.332}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.671}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.71}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:17:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.091}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.064}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.208}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.135}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.742}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.831}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.325}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:18:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.826}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.557}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.54}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.917}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.864}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.017}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.453}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:19:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.194}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.467}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.726}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.164}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.553}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.068}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 483.308}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:20:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.621}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.212}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.105}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.198}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.139}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.126}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.016}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:21:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.904}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.013}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.85}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.758}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.854}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.04}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:22:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.914}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 466.475}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 463.719}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 454.14}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 457.094}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 447.829}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 461.401}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:23:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.825}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 290.458}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 287.787}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 265.354}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 280.842}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.606}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 278.024}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:24:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.929}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 232.377}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.986}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.618}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 223.566}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 228.343}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 229.739}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:25:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.993}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 313.11}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 321.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 327.008}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 296.129}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 314.932}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 304.736}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:26:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.866}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.499}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.019}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.665}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.403}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.38}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 484.052}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:27:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.989}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.577}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.339}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.66}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.335}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.159}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.631}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:28:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.86}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.738}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.852}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.238}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.86}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.873}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.156}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:29:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.006}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.577}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.181}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.281}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.513}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.35}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.439}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:30:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.643}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.9}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.791}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.032}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.294}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.549}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.492}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:31:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.882}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 381.425}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 365.766}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 381.042}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 366.627}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.57}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 349.01}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:32:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 231.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.305}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.245}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 212.716}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.394}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 215.306}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:33:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.931}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 305.457}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 296.197}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 299.107}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 288.084}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 295.245}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 283.962}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:34:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.908}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 470.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.553}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 496.412}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.606}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.053}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 465.367}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:35:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.946}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.576}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.933}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.397}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.841}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.484}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:36:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.928}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.539}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.988}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.761}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.472}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.973}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:37:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.899}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.619}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.967}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.965}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.326}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.923}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.761}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:38:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.68}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.634}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.317}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.548}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.256}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.125}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.46}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:39:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.831}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 370.897}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 366.67}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 351.845}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 367.621}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 386.229}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 368.536}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:40:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.066}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 272.482}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 283.692}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 277.09}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 269.63}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 269.692}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 279.281}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:41:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.767}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 407.894}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 390.305}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 373.091}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 367.886}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 389.619}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 380.304}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:42:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.969}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 491.632}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 459.037}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 490.594}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:43:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.962}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.602}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.844}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.093}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.49}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.74}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.933}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:44:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.841}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.78}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.793}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.029}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.394}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.43}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.278}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:45:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.904}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.564}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.331}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.607}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.06}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.132}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 467.952}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:46:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.944}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.205}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.734}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.545}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.528}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.676}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.265}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:47:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.914}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.587}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.102}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.373}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.791}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.914}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.986}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:48:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.913}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 378.953}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 372.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 385.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 367.246}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 390.256}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 364.882}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:49:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.802}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 228.673}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.614}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.765}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 219.533}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 224.721}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 224.448}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:50:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.768}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 373.002}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 366.492}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.094}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 384.584}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.973}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 353.31}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:51:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.895}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.382}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.076}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.697}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.77}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.686}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 492.207}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:52:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.918}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.135}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.648}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.726}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.633}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.137}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.514}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:53:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.776}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 476.701}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.805}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.654}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.664}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.966}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.659}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:54:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.822}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.965}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.809}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.374}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.288}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.256}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:55:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.896}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 281.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 267.616}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 273.132}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 256.149}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 260.844}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 280.248}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:56:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.957}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 252.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 243.554}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 229.021}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 226.613}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 244.619}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 212.54}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:57:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.716}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 350.398}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 334.481}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 346.251}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 341.218}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 363.211}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 363.165}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:58:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.794}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 468.279}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.243}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.952}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 487.69}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.888}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 462.414}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 18:59:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.854}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.616}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.955}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.718}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.973}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:00:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.68}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.421}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.714}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.997}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.514}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 478.669}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:01:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.873}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.224}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.29}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.253}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.073}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.146}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.415}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:02:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.799}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.616}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.907}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.205}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.394}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.746}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:03:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.954}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 427.283}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 440.338}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 432.686}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 433.136}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 444.958}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 445.693}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:04:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.907}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 243.126}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 260.713}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 230.247}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 244.714}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 261.642}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 248.575}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:05:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.664}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 397.803}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 387.341}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 354.902}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 389.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 372.384}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 390.004}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:06:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.763}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.818}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.4}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.709}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.966}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.79}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 486.615}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:07:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.728}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 490.699}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 491.626}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 489.219}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 486.141}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.677}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 488.465}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:08:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.982}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 489.74}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.405}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.903}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.394}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 484.192}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:09:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.961}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.393}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.029}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.057}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.514}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.247}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.114}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:10:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.875}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.32}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.736}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.08}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.305}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.186}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.885}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:11:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.93}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.186}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.528}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.38}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.62}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.987}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.822}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:12:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.938}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 333.902}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 310.689}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 322.351}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 295.251}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 319.666}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 307.455}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:13:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.857}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 221.203}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.659}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 215.502}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 219.834}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.23}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 224.47}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:14:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.919}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.407}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.289}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 486.203}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 489.144}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 486.397}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:15:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.606}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 424.382}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 411.367}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 417.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 403.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 415.051}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 422.464}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:16:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.749}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.332}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.913}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.757}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.097}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.307}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.764}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:17:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.909}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.643}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.685}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.562}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.186}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:18:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.944}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.296}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.894}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 464.459}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.433}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.951}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.95}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:19:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.932}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.841}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.746}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.866}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.54}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.043}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.7}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:20:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.009}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 387.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 393.955}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 395.263}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 385.056}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 399.23}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 404.631}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:21:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.892}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 244.144}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 243.56}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 237.824}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 233.065}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 236.153}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 240.597}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:22:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.988}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 358.731}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.709}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.823}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 378.953}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 385.087}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 371.103}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:23:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.947}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.306}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.34}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.612}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:24:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.701}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 489.431}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.408}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.576}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.436}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.98}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 478.285}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:25:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.89}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.241}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.36}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.482}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.209}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.619}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.19}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:26:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.982}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.385}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.191}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.504}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.974}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.809}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.098}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:27:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.91}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.6}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.164}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.525}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.907}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.647}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.867}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:28:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.68}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.964}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.777}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.814}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.41}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.312}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:29:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.311}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.102}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.915}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.573}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.393}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:30:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.954}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 349.284}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 331.691}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 327.267}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 323.724}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 346.703}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 341.322}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:31:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.836}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 227.286}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 232.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.104}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 217.641}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 233.696}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 218.559}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:32:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 465.847}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 451.588}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 439.769}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 459.612}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 431.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 435.468}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:33:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.765}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 443.699}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 452.531}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 440.151}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 454.816}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 456.486}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 453.446}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:34:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.85}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.479}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.899}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.224}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.4}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.39}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.836}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:35:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.637}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.416}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.263}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.349}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.09}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.231}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.93}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:36:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.934}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.657}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.212}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.844}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.343}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.03}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:37:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.877}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.725}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.499}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.059}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.837}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.865}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.956}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:38:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.881}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.421}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.255}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.09}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.417}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.293}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.877}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:39:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.849}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 456.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 444.968}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 444.753}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 436.428}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 453.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 459.476}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:40:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.037}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 306.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 269.91}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 296.695}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 263.933}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 292.271}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 270.926}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:41:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.915}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 228.834}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.875}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.552}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 216.35}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.856}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 222.577}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:42:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.861}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 284.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 270.883}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 280.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 285.03}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 267.709}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 284.306}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:43:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.774}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.1}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.908}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.117}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.09}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.164}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.329}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:44:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.035}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.34}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.628}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.888}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.753}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.046}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.858}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:45:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.998}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.129}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.347}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.229}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.488}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.157}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.602}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:46:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.917}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.228}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.662}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.658}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.32}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.0}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.429}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:47:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.81}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 400.935}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.775}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 395.539}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 394.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 391.651}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 393.85}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:48:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.989}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 247.455}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 245.678}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 232.376}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 229.729}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 241.47}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 233.182}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:49:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.783}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 383.467}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 363.702}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.733}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 365.838}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 369.262}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 373.19}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:50:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.95}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 489.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.698}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.07}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.909}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.371}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 485.106}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:51:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.95}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 147.77}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 150.156}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 146.77}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 141.526}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 154.155}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 144.849}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 19:52:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.289}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 139.558}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 142.321}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 139.342}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 132.658}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.856}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.957}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:53:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.214}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 139.074}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 142.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.575}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 132.221}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.576}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.724}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:54:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.844}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 138.812}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 142.023}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.503}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.952}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.639}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.545}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:55:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.573}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.622}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.639}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.924}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.544}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.5}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:56:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.514}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.524}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.65}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.788}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.765}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.402}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.429}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:57:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.283}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.418}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.557}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.568}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.855}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.337}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.448}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:58:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.478}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.48}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.643}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.794}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.834}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.376}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.422}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 19:59:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.847}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.197}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.16}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.864}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.878}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.834}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.693}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:00:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.872}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.642}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.7}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.099}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.506}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.003}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:01:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.158}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.2}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.096}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.979}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.201}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.362}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:02:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.172}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.597}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.37}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.867}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.833}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.89}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.916}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:03:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.283}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.98}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.919}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.106}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.158}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.311}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.673}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:04:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.205}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.096}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.289}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.152}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.094}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.977}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.048}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:05:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.602}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.535}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.498}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.901}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.235}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.852}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:06:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.542}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.515}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.661}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.359}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.518}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.953}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.377}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:07:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.777}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 385.218}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 373.342}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 381.071}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 365.419}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 386.58}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 379.406}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:08:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.721}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 318.396}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 297.547}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 309.908}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 292.254}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 328.632}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 307.882}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:09:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.666}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 399.208}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 411.739}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 402.326}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 397.874}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 411.098}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 412.935}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:10:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.614}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 446.601}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 457.41}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 446.324}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 454.627}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 446.464}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:11:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.694}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.06}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.495}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.117}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.46}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.1}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.189}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:12:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.71}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.265}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.23}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.852}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.247}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.835}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.27}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:13:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.706}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.457}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.403}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.183}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.247}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.451}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:14:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.641}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.583}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.986}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.569}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.423}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.422}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.32}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:15:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.617}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.416}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.384}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.021}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.859}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.548}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.088}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:16:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.7}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.165}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.684}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.889}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.755}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.806}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.025}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:17:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.626}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.602}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.324}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.632}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.207}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.997}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:18:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.936}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 228.167}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 232.668}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 240.568}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 221.775}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 233.548}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 219.832}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:19:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.789}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.903}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.723}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.53}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 485.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 486.228}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:20:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.344}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.622}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.38}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.647}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.659}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:21:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.885}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.541}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.392}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.493}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.279}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:22:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.099}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.798}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.194}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.576}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.216}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.586}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:23:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.745}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.369}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.911}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.8}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.194}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.131}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:24:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.895}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 433.521}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 410.128}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.526}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 415.026}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.894}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 418.871}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:25:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.791}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 227.975}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 231.738}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.264}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 217.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 229.792}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 221.07}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:26:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.951}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 474.686}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 460.773}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 454.283}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 434.304}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 438.632}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 467.432}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:27:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.983}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.732}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.728}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.625}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.967}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.626}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:28:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.986}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.983}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.22}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.104}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.276}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.883}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.858}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:29:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.011}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.194}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.197}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.902}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.76}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.708}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.809}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:30:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.899}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 281.643}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 266.611}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 265.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 252.217}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 286.302}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 278.592}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:31:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.877}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 309.278}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 302.241}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 281.919}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 308.468}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 293.05}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 298.352}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:32:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.748}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 311.112}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 307.741}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 299.324}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 302.775}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 304.541}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 307.819}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:33:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.027}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 490.79}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.474}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 487.283}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.37}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 486.64}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:34:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.685}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.039}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.178}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.132}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.86}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.745}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.027}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:35:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.982}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.63}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.245}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.571}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.994}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.991}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.73}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:36:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.805}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.717}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.208}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.804}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.185}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.896}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.98}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:37:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.015}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.384}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.453}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.182}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.963}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.339}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:38:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.791}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 461.204}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 421.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 436.744}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 419.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 441.772}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 425.228}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:39:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.966}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 415.802}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 381.894}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 405.24}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 381.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 394.825}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 383.68}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:40:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.686}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 258.057}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 245.402}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 256.961}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 242.167}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 245.676}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 257.567}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:41:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.922}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 285.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 290.904}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 302.22}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 292.7}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 288.287}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 279.476}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:42:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.117}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 392.583}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 377.001}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 353.952}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 377.693}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 359.494}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 371.231}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:43:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.202}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.005}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.65}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.665}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.023}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.275}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.978}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:44:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.915}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.958}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.989}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.142}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 463.674}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.485}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.413}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:45:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.972}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.693}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.156}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.548}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.528}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.239}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:46:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.884}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.547}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.751}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.544}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.882}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.284}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.886}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:47:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.975}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 414.288}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 408.383}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 410.413}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 408.445}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 400.638}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 399.048}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:48:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.116}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 223.06}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.921}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 233.103}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.355}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 221.509}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:49:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.962}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 304.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 293.217}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 270.46}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 282.24}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 286.671}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 263.391}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:50:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.697}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 492.572}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.605}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.672}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.952}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.294}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 488.519}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:51:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.109}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.022}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.745}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.255}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.309}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 478.741}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:52:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.977}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.171}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.003}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.984}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.499}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.719}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.354}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:53:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.968}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.799}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.623}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.32}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.447}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.136}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:54:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.004}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.563}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.987}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.927}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.789}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.574}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:55:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.916}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.611}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.723}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.36}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.585}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.311}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:56:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.951}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 353.596}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 344.555}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 346.582}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 340.697}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 353.402}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 363.866}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:57:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.831}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 357.274}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.27}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 351.94}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 365.556}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 348.579}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 357.882}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:58:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.778}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 494.712}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.145}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.566}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 483.355}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 491.129}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 495.079}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 20:59:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.929}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.162}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.974}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.543}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.954}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.476}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 486.562}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:00:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.084}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.499}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.329}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.413}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.208}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.158}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.375}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:01:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.118}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.115}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.312}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.201}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.778}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.462}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 466.171}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:02:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.029}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.072}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.257}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.715}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.871}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.165}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:03:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.878}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 469.407}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.543}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 463.083}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 464.753}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.464}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 467.937}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:04:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.991}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 339.044}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 347.094}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 351.085}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 344.48}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 367.833}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 353.326}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:05:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.037}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 290.639}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 284.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 276.944}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 279.525}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 296.315}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 267.221}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:06:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.947}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 375.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 393.002}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 383.525}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 369.948}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 396.984}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 377.887}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:07:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.094}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 433.053}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 434.299}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 405.886}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 407.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 423.268}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 408.606}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:08:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.007}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.623}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.86}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.653}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.52}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.561}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 483.306}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:09:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.871}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.55}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.493}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.698}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.656}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.549}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:10:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.013}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.959}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.583}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.087}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.211}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.898}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.728}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:11:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.094}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 367.494}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.654}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 364.927}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 383.263}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 364.461}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:12:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.972}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 244.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 236.813}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 243.789}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 253.685}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.807}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 250.995}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:13:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.929}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 360.117}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 342.721}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 354.321}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 349.894}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 355.918}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 351.216}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:14:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.087}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 467.056}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 452.353}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 459.855}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 447.331}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 446.719}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 451.802}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:15:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.057}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.557}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.61}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.191}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.528}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.566}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:16:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.098}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.275}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.653}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.659}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.892}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.499}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.931}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:17:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.763}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.065}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.483}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.045}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.542}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.974}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.722}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:18:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.037}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.722}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.59}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.515}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.86}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.623}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.544}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:19:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.09}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 452.103}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.635}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 431.524}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 437.236}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 437.57}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 448.605}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:20:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.864}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 223.238}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.519}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 208.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 212.255}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.525}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 213.754}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:21:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.054}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 432.503}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 413.068}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 430.413}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 420.255}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 418.213}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 439.777}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:22:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.847}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 491.438}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.93}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.541}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 485.371}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 483.091}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:23:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.046}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.205}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.986}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.493}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.316}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.828}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.904}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:24:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.862}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.066}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.766}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.99}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.713}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.251}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 466.042}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:25:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.996}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.564}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.769}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.713}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.627}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.541}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.476}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:26:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.897}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.728}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.621}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.491}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.965}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.859}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.402}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:27:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.796}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.831}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.413}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.154}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.854}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.768}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.408}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:28:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.09}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 262.509}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 226.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 250.663}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 253.516}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 256.361}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 232.225}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:29:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.013}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 263.015}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 253.51}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 255.768}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 251.492}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 255.573}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 255.62}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:30:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.832}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 308.8}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 320.554}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 295.414}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 286.471}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 292.744}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 309.504}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:31:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 237.485}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 254.297}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 236.552}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 213.476}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 230.687}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 228.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:32:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.55}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 411.243}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 403.496}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 383.819}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 406.211}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.492}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 412.552}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:33:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.772}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.549}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.175}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.537}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.231}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.527}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:34:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.82}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.87}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.917}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.521}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.653}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.225}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.62}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:35:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.398}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.689}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.566}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.531}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.188}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:36:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.975}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.845}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.894}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.861}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.579}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.903}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.908}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:37:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.862}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.373}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.575}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.761}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.959}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.719}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.614}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:38:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.98}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.745}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.193}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.941}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.981}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.964}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:39:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.86}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.516}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.782}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.173}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:40:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.945}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 268.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 276.674}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 248.473}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 248.562}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 283.807}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 261.261}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:41:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.935}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 302.823}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 288.236}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 275.08}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 283.705}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 287.006}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 289.324}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:42:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.885}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.288}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.054}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.217}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.213}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.299}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:43:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.943}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.763}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.586}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.227}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:44:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.759}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.986}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.693}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.539}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.773}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.488}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.159}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:45:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.908}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.411}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.227}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.445}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.332}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.276}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.397}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:46:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.881}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.06}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.295}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.55}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.423}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.266}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:47:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.875}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 404.361}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 391.175}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.602}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 391.527}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 398.258}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 404.195}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:48:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.001}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 229.629}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.91}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.289}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 232.902}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 227.414}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 218.537}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:49:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.098}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 257.752}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 261.988}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 257.995}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 251.41}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 260.562}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 254.116}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:50:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.917}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 451.528}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 426.218}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 430.051}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 431.325}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 450.295}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 415.434}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:51:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.011}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.595}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.629}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.729}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.851}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.221}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:52:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.896}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.589}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.719}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.575}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.221}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.823}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.409}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:53:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.117}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.053}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.224}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.725}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.623}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.264}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.652}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:54:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.881}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.01}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.009}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.813}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.843}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.165}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.238}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:55:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.277}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.904}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.097}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.125}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.44}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.794}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:56:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.716}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.876}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.656}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.675}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.858}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.475}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:57:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.978}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 387.254}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 375.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 369.402}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 380.012}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 391.407}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 387.198}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:58:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 376.575}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 386.692}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 378.466}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 387.215}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 374.372}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 381.016}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 21:59:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.671}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 311.45}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 300.034}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 284.671}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 280.042}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 295.471}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 278.23}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:00:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.762}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 295.521}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 301.259}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.681}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 299.474}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 278.956}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 276.168}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:01:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.81}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.897}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.661}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.577}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.106}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.561}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.263}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:02:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.944}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.533}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.549}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.242}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.898}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.571}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.934}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:03:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.985}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.386}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.996}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.704}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.714}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.973}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.935}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:04:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.057}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.012}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.968}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.167}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.61}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.28}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.375}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:05:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.888}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.183}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.709}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.037}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.16}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.282}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:06:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.84}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.432}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.882}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.161}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.681}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.789}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.183}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:07:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.867}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 291.684}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 270.442}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 267.082}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 251.311}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 279.065}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 282.097}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:08:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.716}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 228.45}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.868}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.897}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 214.39}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 233.065}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 215.613}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:09:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.002}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 428.418}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.332}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 425.437}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 419.829}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 423.591}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 434.052}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:10:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.785}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.574}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.812}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.782}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.877}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.385}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.82}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:11:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.763}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.114}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.696}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.199}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.159}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.478}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 484.439}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:12:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.69}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.383}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.158}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.131}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.305}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.403}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.894}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:13:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.794}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.146}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.96}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.255}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.216}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.917}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.716}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:14:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.949}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.498}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.759}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.183}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.306}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.259}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.877}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:15:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.7}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.738}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.526}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.44}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.011}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.846}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:16:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.769}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 457.224}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 445.569}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 455.731}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 445.237}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 444.356}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 450.75}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:17:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.909}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 249.08}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 251.757}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 236.572}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 262.291}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 257.09}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 253.049}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:18:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.778}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 232.613}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 231.741}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 242.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 221.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.199}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 226.623}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:19:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.846}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 419.5}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 419.372}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 411.721}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 416.847}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 431.673}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 447.522}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:20:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.828}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 440.637}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 451.261}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 457.638}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 464.495}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 442.709}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 450.03}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:21:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.773}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.227}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.675}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.914}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 487.811}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.599}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 491.424}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:22:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.764}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.403}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.343}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.698}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.57}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.716}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.981}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:23:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.84}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.205}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.243}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.864}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.799}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.698}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.339}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:24:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.85}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.333}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.438}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.694}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.603}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.441}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:25:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.839}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.383}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.302}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.105}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.663}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.792}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.637}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:26:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.96}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 312.508}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 293.28}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 317.86}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 309.961}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 314.424}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 299.276}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:27:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.761}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 247.477}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 251.164}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 220.424}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 242.561}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 222.923}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 225.731}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:28:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.721}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 293.444}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.249}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 287.933}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 268.441}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 293.457}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 272.574}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:29:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.836}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.427}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.782}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.866}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.331}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.385}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 487.819}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:30:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.946}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.412}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.432}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.324}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.81}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.449}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.692}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:31:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.9}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.097}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.37}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.959}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.683}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.701}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:32:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.772}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.629}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.554}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.131}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.663}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.326}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:33:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.863}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 432.314}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 445.885}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 435.472}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 426.625}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 449.699}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 438.923}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:34:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.806}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 307.018}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 323.867}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 308.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 295.381}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 326.151}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 317.332}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:35:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.839}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 334.469}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 328.735}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 331.474}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 333.847}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 318.959}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 317.077}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:36:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.884}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 490.28}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.234}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.06}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.606}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 496.04}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 490.84}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:37:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.86}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.002}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.082}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.016}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.867}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.189}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.881}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:38:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.956}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 476.816}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.632}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.116}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.484}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.123}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:39:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.929}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.962}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.037}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.287}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.861}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.857}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:40:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.783}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 309.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 298.071}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 280.531}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 305.298}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 304.211}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 318.723}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:41:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.979}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 311.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 323.775}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 310.807}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 313.358}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 297.569}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 310.222}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:42:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.675}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.641}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.389}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.606}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.071}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.266}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:43:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.786}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.07}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.985}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.93}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.883}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.179}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.656}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:44:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.584}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.5}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.781}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 466.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.85}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.502}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.96}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:45:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.703}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.385}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.858}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.572}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.578}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.808}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.693}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:46:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.634}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 451.354}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 444.48}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 449.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 429.576}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 428.536}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 449.975}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:47:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.816}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 263.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 254.291}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.684}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 247.449}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 264.151}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 271.569}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:48:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.679}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 296.365}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 298.064}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 297.201}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 286.774}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 273.437}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 297.168}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:49:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.636}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.559}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.473}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.744}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.976}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.436}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 482.398}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:50:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.793}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.278}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.328}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.532}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.304}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.89}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:51:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.705}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.346}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.05}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.339}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.689}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.247}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.963}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:52:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.741}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.238}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.78}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.326}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.138}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.383}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.796}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:53:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.934}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 317.422}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 310.935}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 312.699}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 317.054}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 312.35}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 312.666}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:54:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.698}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 215.007}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 212.802}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.226}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 208.639}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 209.517}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 213.215}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:55:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.68}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.787}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.903}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.042}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 484.338}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 492.27}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.374}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:56:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.679}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 491.982}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.272}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.094}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.248}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 485.598}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:57:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.64}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.281}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.179}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.227}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.102}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.326}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.306}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:58:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.762}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.027}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.595}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 464.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.516}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.219}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.572}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 22:59:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.89}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.489}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.53}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.148}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.096}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:00:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.113}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 334.525}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 336.269}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 332.913}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 351.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 335.729}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 339.644}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:01:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.839}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 250.228}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.567}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 241.735}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 254.134}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 245.425}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 271.986}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:02:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.94}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 402.24}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 393.154}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 399.102}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 379.074}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 367.536}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 377.056}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:03:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.742}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.0}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 148.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 143.662}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 137.951}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.193}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 140.34}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:04:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 474.398}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 461.2}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 475.7}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.406}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 490.404}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:05:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.465}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.601}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.368}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.154}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 481.26}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.054}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.877}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 23:06:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.737}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.376}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884
Reference in New Issue View Git Blame Copy Permalink