p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity
LLaMA-Factory-Mirror/results/gpu_status/gpu_status_0825.json

133 lines
152 KiB
JSON
Raw Blame History

{"cur_time": "2024-08-25 13:02:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 492.358}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.10888671875, "powerusage_W": 138.329}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 140.951}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 138.643}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 130.779}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 136.631}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 135.738}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:03:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.45}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.10888671875, "powerusage_W": 138.356}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 140.939}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 138.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 130.848}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 136.585}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 135.791}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:04:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.988}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.10888671875, "powerusage_W": 138.332}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 140.938}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 138.604}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 130.795}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 136.62}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.22064208984375, "powerusage_W": 135.746}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:05:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 490.328}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.369}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 140.899}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.691}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.914}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.628}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.825}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:06:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 493.174}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.394}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 140.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.674}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.913}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.454}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.871}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:07:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 490.915}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.44}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 140.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.731}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.927}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.518}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.878}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:08:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.096}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.518}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 141.022}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.633}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.959}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.633}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.914}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:09:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.382}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.345}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 140.977}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.579}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.822}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.512}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.808}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:10:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.241}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.74127197265625, "powerusage_W": 138.4}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 141.002}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 138.588}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 130.893}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 136.401}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.85302734375, "powerusage_W": 135.887}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:11:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 485.231}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 138.54}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.721}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.041}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:12:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.657}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 138.361}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 140.968}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.642}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 130.809}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.633}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 135.835}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:13:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 484.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.3736572265625, "powerusage_W": 138.525}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.778}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.022}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.642}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 135.989}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:14:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.395}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.598}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.175}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.803}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.077}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.68}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.048}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:15:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.388}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.515}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 140.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.016}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.683}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 135.949}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:16:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 481.856}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.57}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.182}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.726}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.11}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.717}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.077}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:17:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.009}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 3.4342041015625, "powerusage_W": 138.567}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 141.201}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 138.848}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 131.061}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.852}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.48541259765625, "powerusage_W": 136.054}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}]}
{"cur_time": "2024-08-25 13:18:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.206}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.1470947265625, "powerusage_W": 221.117}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.33111572265625, "powerusage_W": 230.713}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33111572265625, "powerusage_W": 256.256}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.25103759765625, "powerusage_W": 226.239}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.33111572265625, "powerusage_W": 237.734}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09674072265625, "powerusage_W": 232.247}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.23828125}]}
{"cur_time": "2024-08-25 13:19:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.057}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 238.304}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 268.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 252.611}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 253.838}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 234.69}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 266.429}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:20:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 493.352}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 274.847}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 263.303}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 253.489}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 241.529}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 252.73}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 255.432}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:21:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 487.503}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 373.263}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 381.024}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 385.971}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 399.252}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 384.945}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 408.059}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:22:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 482.199}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 413.892}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 406.436}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 404.415}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 423.189}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 407.281}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 411.111}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:23:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 484.292}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.715}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.127}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.854}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.416}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 488.671}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:24:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 493.669}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.675}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.233}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.845}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.309}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.345}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.999}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:25:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 484.702}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.364}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.616}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.428}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 473.191}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.214}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:26:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.38}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 479.389}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.779}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.079}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.802}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.013}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:27:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 490.001}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.809}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.273}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.427}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.377}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.858}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.429}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:28:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 484.635}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.671}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.938}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.873}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.301}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.772}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.052}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:29:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.927}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.176}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.582}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.549}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.819}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.242}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:30:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 493.898}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.61}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.826}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.823}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 468.565}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:31:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 480.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 373.325}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 352.571}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 352.77}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 367.177}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 366.063}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 354.494}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:32:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.511}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 217.335}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.746}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 214.636}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 211.027}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.896}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 225.177}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:33:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 487.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 226.198}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 237.764}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 210.047}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 222.538}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.316}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 231.238}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:34:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 486.395}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 233.613}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 242.198}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 229.961}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 240.821}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 230.615}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 242.197}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:35:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 490.448}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 460.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.207}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.658}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 459.364}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.732}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 480.532}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:36:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.362}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 493.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.345}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.362}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 491.608}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 488.97}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:37:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 485.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.728}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.918}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.886}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.704}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 476.674}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:38:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 487.575}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.261}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.502}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.955}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 476.167}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.47}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 479.474}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:39:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.256}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.529}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.524}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.403}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 464.026}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.182}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:40:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 482.638}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.452}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.196}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.89}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.978}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.914}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:41:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 485.177}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.605}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.913}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.715}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.707}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.409}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:42:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 491.808}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.151}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.033}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.948}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.062}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.308}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.847}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:43:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 482.02}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 377.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 382.553}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 371.14}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 381.732}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 389.34}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 389.693}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:44:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.749}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 340.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 327.94}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 353.07}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 331.983}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 356.889}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 347.341}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:45:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.154}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 275.526}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 273.01}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 284.112}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 250.205}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.304}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 285.96}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:46:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 485.856}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 348.672}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 376.36}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 368.998}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 364.542}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 375.496}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 384.474}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:47:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.692}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.271}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 486.24}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.381}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.81}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.96}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 485.813}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:48:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 485.325}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.405}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.956}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.14}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 479.823}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.602}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.931}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:49:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 492.434}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.348}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.85}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.122}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.409}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.102}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 471.765}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:50:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 492.952}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.823}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.588}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.682}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 472.636}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.579}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.836}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:51:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 480.806}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.077}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.723}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.243}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.604}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.576}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:52:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.016}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.693}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.123}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.844}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.822}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.043}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.726}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:53:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 480.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.2}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.456}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.537}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 468.516}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.404}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.232}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:54:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.177}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 306.35}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 286.76}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 263.039}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 281.539}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.573}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 280.884}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:55:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.665}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 234.102}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 227.206}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.349}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 217.946}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 218.246}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 214.497}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:56:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 484.108}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 314.58}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 344.547}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 340.634}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 330.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 334.259}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 319.871}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:57:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 492.747}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.987}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.207}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 469.066}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 488.279}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 467.419}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 465.103}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:58:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 491.726}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.581}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.762}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 487.911}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 488.575}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 487.855}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 13:59:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 481.72}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.651}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.283}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.677}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.846}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.616}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:00:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 488.383}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.366}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.047}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.578}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.359}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.897}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 475.025}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:01:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 492.344}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 486.375}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.593}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.743}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 474.107}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.928}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.82}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:02:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 483.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.583}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.581}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.64}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.441}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.125}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 474.355}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:03:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.07135009765625, "powerusage_W": 489.12}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.67}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.335}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.065}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 465.688}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.287}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.539}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:04:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.80029296875, "powerusage_W": 383.853}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 484.783}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.378}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.03}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.176}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.756}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.075}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:05:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 64.60064697265625, "powerusage_W": 369.7}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.055}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.403}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.569}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 470.566}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.601}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.135}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:06:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 367.924}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.65}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 480.145}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.747}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 478.444}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.194}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 477.803}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:07:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 379.713}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 490.001}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.678}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 487.622}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 488.414}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.495}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 490.055}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:08:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 376.849}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 437.512}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 440.757}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 439.657}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 438.237}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 453.855}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 457.449}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:09:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 373.116}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.664}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 482.134}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.541}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 477.664}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.204}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.586}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:10:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 383.98}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 477.041}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.753}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 465.712}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 464.906}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.839}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.529}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:11:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 388.144}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 480.973}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.834}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.896}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.559}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.677}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.25}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:12:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.9830322265625, "powerusage_W": 380.9}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 483.501}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 471.698}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 472.003}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.377}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.322}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.741}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:13:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.9830322265625, "powerusage_W": 372.925}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 478.55}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.902}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 460.916}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 459.848}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.953}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.013}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:14:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.9830322265625, "powerusage_W": 389.492}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 488.61}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.525}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 466.315}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.202}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.54}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:15:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.9830322265625, "powerusage_W": 386.905}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.974}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.665}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 467.301}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.346}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 469.71}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:16:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.9830322265625, "powerusage_W": 368.725}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 481.8}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 477.687}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 470.0}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.736}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.173}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 472.986}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:17:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 371.757}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 421.126}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 431.829}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 427.1}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 437.506}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 424.053}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 434.071}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:18:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 391.6}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 417.367}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 447.096}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 404.054}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 412.585}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 449.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 417.01}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:19:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 383.449}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.266}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 474.425}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 457.895}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.557}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.875}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 448.679}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:20:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 378.078}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 405.804}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 383.061}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 391.02}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 370.481}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 403.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 391.614}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:21:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 387.377}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 229.637}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 219.453}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 221.38}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 213.879}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 216.931}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 212.464}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:22:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 76.1646728515625, "powerusage_W": 379.838}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 346.754}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 327.446}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 336.523}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7662353515625, "powerusage_W": 330.725}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 340.616}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 349.505}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 1653855, "create_time": 1724595725.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-25 14:23:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.6392822265625, "powerusage_W": 376.098}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.88494873046875, "powerusage_W": 260.91}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.0279541015625, "powerusage_W": 269.426}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 249.209}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.7916259765625, "powerusage_W": 258.518}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 265.165}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.3814697265625, "powerusage_W": 265.159}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 1653855, "create_time": 1724595725.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.27734375}]}
{"cur_time": "2024-08-25 14:24:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 363.913}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 485.084}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 476.324}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 468.835}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 469.688}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.562}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 473.491}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:25:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 364.982}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 482.801}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 475.785}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 473.277}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 471.227}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 481.729}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 470.822}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:26:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 70.21978759765625, "powerusage_W": 384.556}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 394.14}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 383.714}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 388.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 385.239}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 394.096}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 390.841}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:27:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 148.546}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 463.537}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 463.334}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 425.762}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 427.729}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 464.401}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 431.864}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:28:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.19049072265625, "powerusage_W": 378.244}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.68963623046875, "powerusage_W": 485.485}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 473.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 471.97}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.7916259765625, "powerusage_W": 469.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 479.589}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.3814697265625, "powerusage_W": 470.395}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 1239772, "create_time": 1724596043.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.27734375}]}
{"cur_time": "2024-08-25 14:29:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.81353759765625, "powerusage_W": 380.863}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.07049560546875, "powerusage_W": 462.894}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.2545166015625, "powerusage_W": 455.424}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.2545166015625, "powerusage_W": 416.62}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.1744384765625, "powerusage_W": 417.728}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.2545166015625, "powerusage_W": 456.314}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.7252197265625, "powerusage_W": 450.673}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 1239772, "create_time": 1724596043.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.619140625}]}
{"cur_time": "2024-08-25 14:30:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19287109375, "powerusage_W": 383.614}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 493.137}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 484.883}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.848}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 482.195}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 485.796}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 484.013}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:31:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19287109375, "powerusage_W": 390.695}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 487.59}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 479.506}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 478.095}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 480.372}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 483.171}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 481.173}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:32:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.47564697265625, "powerusage_W": 380.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.68963623046875, "powerusage_W": 486.601}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 479.282}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 480.325}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.7916259765625, "powerusage_W": 478.854}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.4263916015625, "powerusage_W": 479.635}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 478.598}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 4098936, "create_time": 1724596294.28, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-25 14:33:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.61236572265625, "powerusage_W": 379.455}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.88494873046875, "powerusage_W": 366.572}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.0670166015625, "powerusage_W": 361.963}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.96893310546875, "powerusage_W": 400.907}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.9869384765625, "powerusage_W": 359.418}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.0084228515625, "powerusage_W": 365.566}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.3814697265625, "powerusage_W": 351.325}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 4098936, "create_time": 1724596294.28, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.27734375}]}
{"cur_time": "2024-08-25 14:34:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.11431884765625, "powerusage_W": 365.637}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 371.652}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 370.709}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.3248291015625, "powerusage_W": 409.547}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 363.408}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 367.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 354.414}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:35:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.90142822265625, "powerusage_W": 384.39}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 386.989}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 375.462}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.3248291015625, "powerusage_W": 413.661}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 367.833}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 377.843}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 367.285}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:36:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 75.9029541015625, "powerusage_W": 377.389}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 196.875}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 191.177}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.83807373046875, "powerusage_W": 311.064}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7662353515625, "powerusage_W": 185.731}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 198.182}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 193.988}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 2328827, "create_time": 1724596566.7, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-25 14:37:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 77.3209228515625, "powerusage_W": 376.15}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.88494873046875, "powerusage_W": 372.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.9888916015625, "powerusage_W": 364.073}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.86346435546875, "powerusage_W": 403.166}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.7916259765625, "powerusage_W": 354.127}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 362.173}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.3814697265625, "powerusage_W": 356.975}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 2328827, "create_time": 1724596566.7, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.27734375}]}
{"cur_time": "2024-08-25 14:38:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.90142822265625, "powerusage_W": 409.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 368.305}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 365.045}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 407.664}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 357.661}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 363.327}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 354.396}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:39:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 77.2037353515625, "powerusage_W": 384.081}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.68963623046875, "powerusage_W": 338.587}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 325.453}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.38494873046875, "powerusage_W": 367.871}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.92791748046875, "powerusage_W": 358.464}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.8717041015625, "powerusage_W": 336.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.3814697265625, "powerusage_W": 319.3}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3444764, "create_time": 1724596709.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.27734375}]}
{"cur_time": "2024-08-25 14:40:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 77.4322509765625, "powerusage_W": 375.46}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.88494873046875, "powerusage_W": 336.996}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.0670166015625, "powerusage_W": 327.086}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.58026123046875, "powerusage_W": 370.249}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.53143310546875, "powerusage_W": 354.342}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.0670166015625, "powerusage_W": 330.997}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.5181884765625, "powerusage_W": 316.631}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3444764, "create_time": 1724596709.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.4140625}]}
{"cur_time": "2024-08-25 14:41:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.90142822265625, "powerusage_W": 384.315}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 325.76}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 324.904}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 371.709}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 353.383}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 321.643}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 313.831}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:42:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.90142822265625, "powerusage_W": 381.862}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 338.307}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 323.74}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 384.009}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 357.781}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 328.294}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 318.545}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-25 14:43:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.90142822265625, "powerusage_W": 361.162}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 333.366}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 322.818}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 367.575}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 348.645}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 326.521}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3638916015625, "powerusage_W": 380.937}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.259765625}]}
{"cur_time": "2024-08-25 14:44:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 384.096}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 322.353}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 319.767}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 362.302}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 345.595}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.7271728515625, "powerusage_W": 328.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3638916015625, "powerusage_W": 381.785}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.259765625}]}
{"cur_time": "2024-08-25 14:45:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 390.991}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 313.599}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 313.483}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 346.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 336.549}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5279541015625, "powerusage_W": 444.67}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 374.561}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:46:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 383.643}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 310.581}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8267822265625, "powerusage_W": 362.28}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 345.476}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.2154541015625, "powerusage_W": 334.363}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5279541015625, "powerusage_W": 380.007}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 371.347}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:47:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 392.531}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 289.91}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2408447265625, "powerusage_W": 338.495}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.8463134765625, "powerusage_W": 325.825}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 308.448}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.29315185546875, "powerusage_W": 404.787}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 345.397}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:48:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 388.971}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.03338623046875, "powerusage_W": 333.413}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3619384765625, "powerusage_W": 315.62}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.9400634765625, "powerusage_W": 307.633}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 293.026}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32049560546875, "powerusage_W": 366.389}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 327.49}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:49:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 382.537}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.03338623046875, "powerusage_W": 311.284}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3619384765625, "powerusage_W": 309.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.9400634765625, "powerusage_W": 301.081}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 268.142}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32049560546875, "powerusage_W": 347.183}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 301.375}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:50:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.19482421875, "powerusage_W": 388.491}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 347.175}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 347.959}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.9400634765625, "powerusage_W": 320.589}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 319.78}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32049560546875, "powerusage_W": 383.93}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 343.078}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:51:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 146.538}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 322.709}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 318.003}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 310.441}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 297.991}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32049560546875, "powerusage_W": 436.734}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 316.639}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:52:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.992}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 342.769}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 332.787}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 319.898}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 288.925}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32049560546875, "powerusage_W": 364.884}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 329.471}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:53:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.741}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 331.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 319.135}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 298.465}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 287.394}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32244873046875, "powerusage_W": 442.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 316.141}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:54:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.099}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 338.791}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 314.135}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 305.447}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 298.912}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32244873046875, "powerusage_W": 365.19}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 331.675}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:55:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15643310546875, "powerusage_W": 341.58}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 330.206}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 312.847}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 301.88}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32244873046875, "powerusage_W": 372.664}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 331.096}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:56:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.105}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 332.401}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 317.627}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 306.035}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 296.26}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32244873046875, "powerusage_W": 361.525}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 333.311}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:57:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.835}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 331.417}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 316.193}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 308.703}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 309.592}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.32244873046875, "powerusage_W": 375.786}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 337.39}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:58:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 145.054}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 339.977}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.4927978515625, "powerusage_W": 320.457}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.1666259765625, "powerusage_W": 315.383}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 305.721}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 376.497}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 335.339}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 14:59:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.638}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 338.443}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5885009765625, "powerusage_W": 328.714}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.956}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 305.536}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 368.942}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 333.691}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:00:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.372}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 357.063}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5885009765625, "powerusage_W": 335.679}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 274.459}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 311.017}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 377.927}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 345.03}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:01:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.435}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 336.062}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 330.949}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 272.649}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 298.246}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 373.405}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 337.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:02:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.411}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 343.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 325.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 269.63}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 309.427}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 367.759}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 329.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:03:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.414}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 294.333}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 329.236}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 202.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 271.384}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 352.174}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 316.204}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:04:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.571}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 333.843}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 326.971}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 260.252}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 296.837}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 370.609}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 329.843}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:05:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.471}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 353.586}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 335.472}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 271.512}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 315.994}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 379.507}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 356.736}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:06:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.571}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 330.973}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 317.794}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 271.961}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 312.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42010498046875, "powerusage_W": 440.485}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 333.804}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:07:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.432}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 346.611}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 325.545}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 264.319}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 302.171}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 374.325}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 336.981}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:08:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.428}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 269.307}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 327.795}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 165.325}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 217.368}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 332.324}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 281.084}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:09:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.416}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 339.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 322.587}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 267.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 307.205}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 363.298}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 332.058}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:10:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.429}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 353.63}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 330.244}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 277.536}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 315.914}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 430.699}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 342.408}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:11:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.411}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 342.009}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 322.312}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 268.555}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6607666015625, "powerusage_W": 303.04}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 372.619}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 333.853}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:12:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.417}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 334.548}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 314.577}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 257.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.1119384765625, "powerusage_W": 299.236}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 366.543}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 334.679}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:13:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 144.334}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.29705810546875, "powerusage_W": 331.618}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.7369384765625, "powerusage_W": 315.41}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 238.798}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.1119384765625, "powerusage_W": 283.39}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 88.42205810546875, "powerusage_W": 362.665}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3853759765625, "powerusage_W": 312.688}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2477636, "create_time": 1724591445.59, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 754135, "create_time": 1724596940.04, "name": "pt_main_thread", "user": "user", "used_mem_GB": 20.28125}]}
{"cur_time": "2024-08-25 15:14:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.84}, {"device": 1, "total_mem_GB": 95.5771
Reference in New Issue View Git Blame Copy Permalink