p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity
LLaMA-Factory-Mirror/results/gpu_status/gpu_status_0826.json

152 lines
174 KiB
JSON
Raw Blame History

{"cur_time": "2024-08-26 04:11:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.88}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.042}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.503}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.403}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.159}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.2}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:12:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.244}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.612}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.965}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.75}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.61}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 134.688}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:13:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.743}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.084}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.465}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.147}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.547}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.537}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:14:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.289}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 139.57}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.081}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.097}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.033}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.203}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:15:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.75341796875, "powerusage_W": 140.352}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.09002685546875, "powerusage_W": 138.861}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.0201416015625, "powerusage_W": 139.895}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6490478515625, "powerusage_W": 135.639}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.1920166015625, "powerusage_W": 130.611}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.0201416015625, "powerusage_W": 135.445}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.5513916015625, "powerusage_W": 133.725}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 13.447265625}]}
{"cur_time": "2024-08-26 04:16:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.58935546875, "powerusage_W": 341.646}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.20916748046875, "powerusage_W": 336.278}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3970947265625, "powerusage_W": 337.97}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 332.031}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3111572265625, "powerusage_W": 323.555}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3931884765625, "powerusage_W": 324.689}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.9224853515625, "powerusage_W": 325.899}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.818359375}]}
{"cur_time": "2024-08-26 04:17:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.406}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.932}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.174}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 130.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.193}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.57}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:18:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 6.75537109375, "powerusage_W": 140.549}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.26971435546875, "powerusage_W": 138.742}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.7349853515625, "powerusage_W": 140.51}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.4830322265625, "powerusage_W": 135.671}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.4869384765625, "powerusage_W": 130.362}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.5670166015625, "powerusage_W": 135.635}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.0142822265625, "powerusage_W": 134.022}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 4.91015625}]}
{"cur_time": "2024-08-26 04:19:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.49560546875, "powerusage_W": 142.845}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.36737060546875, "powerusage_W": 341.015}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 142.725}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.8463134765625, "powerusage_W": 333.995}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.7662353515625, "powerusage_W": 333.442}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.8463134765625, "powerusage_W": 341.998}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.3775634765625, "powerusage_W": 339.185}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.2734375}]}
{"cur_time": "2024-08-26 04:20:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.178}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.077}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.639}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.499}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.259}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.751}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 134.636}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:21:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 11.86474609375, "powerusage_W": 140.539}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.36932373046875, "powerusage_W": 138.102}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.6666259765625, "powerusage_W": 140.928}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7291259765625, "powerusage_W": 136.087}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.5552978515625, "powerusage_W": 130.159}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7291259765625, "powerusage_W": 134.919}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.2916259765625, "powerusage_W": 135.744}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3538113, "create_time": 1724646019.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.1875}]}
{"cur_time": "2024-08-26 04:22:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.52685546875, "powerusage_W": 347.486}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.20916748046875, "powerusage_W": 346.516}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3970947265625, "powerusage_W": 343.166}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 339.91}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3111572265625, "powerusage_W": 329.189}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3931884765625, "powerusage_W": 335.663}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.9224853515625, "powerusage_W": 333.199}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3538113, "create_time": 1724646019.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.818359375}]}
{"cur_time": "2024-08-26 04:23:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.94677734375, "powerusage_W": 145.015}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.929}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.015}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.763}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.027}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:24:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28076171875, "powerusage_W": 351.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.524}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.42}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.835}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.073}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.755}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:25:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28271484375, "powerusage_W": 346.615}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.66}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.723}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.2}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.329}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.484}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.034}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:26:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28271484375, "powerusage_W": 353.821}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.22}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.444}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.078}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.647}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:27:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.73931884765625, "powerusage_W": 309.121}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.485}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5260009765625, "powerusage_W": 250.536}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.3013916015625, "powerusage_W": 137.83}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.264}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8931884765625, "powerusage_W": 136.795}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.031}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:28:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.73541259765625, "powerusage_W": 302.962}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 281.768}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 328.171}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3072509765625, "powerusage_W": 309.734}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 272.093}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 259.733}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.077}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:29:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.21392822265625, "powerusage_W": 304.392}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 278.596}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 313.838}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 312.159}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 277.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 248.308}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.164}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:30:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.21392822265625, "powerusage_W": 300.442}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 287.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 284.01}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 304.484}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 272.111}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 256.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.777}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:31:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.69439697265625, "powerusage_W": 304.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 285.531}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 287.784}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 313.126}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.573}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.435}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.643}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:32:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.71197509765625, "powerusage_W": 301.677}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 285.758}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 340.98}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 303.952}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.402}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 260.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.513}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:33:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44439697265625, "powerusage_W": 202.796}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 284.235}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 317.666}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 299.932}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 254.694}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.296}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:34:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09478759765625, "powerusage_W": 302.032}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 289.987}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 321.66}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 308.381}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.557}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.443}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:35:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.344}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 290.232}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 314.5}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 304.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.016}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 246.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.446}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:36:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.156}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 284.763}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 306.186}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 320.057}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.883}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 252.547}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.477}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:37:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.467}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 288.747}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 311.333}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 311.475}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.804}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 255.972}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.067}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:38:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 315.185}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 294.833}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 336.552}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 257.864}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.177}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:39:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 301.547}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 283.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 310.811}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 318.432}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.556}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.043}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.426}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:40:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.12799072265625, "powerusage_W": 330.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 291.268}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 301.438}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 297.474}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 284.742}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 264.848}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.815}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:41:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.816}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.087}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 300.562}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.865}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.043}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.918}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.326}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:42:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 298.184}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.347}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 282.174}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 319.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.061}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 258.145}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.145}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:43:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 316.107}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.176}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 323.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 324.684}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 250.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.833}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:44:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 298.994}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 280.52}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 311.194}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 308.844}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 250.865}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.807}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:45:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.21392822265625, "powerusage_W": 296.592}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 283.443}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 295.926}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 296.348}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.179}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 271.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.644}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:46:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.22174072265625, "powerusage_W": 202.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 289.021}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 287.812}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 304.927}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 272.911}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 245.667}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.666}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:47:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09478759765625, "powerusage_W": 299.412}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 285.618}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 297.281}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.333}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.551}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:48:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 296.588}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 275.752}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 322.029}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 305.368}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.571}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 254.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.59}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:49:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 309.065}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 281.992}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 308.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 313.519}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 276.568}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.515}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.473}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:50:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 319.809}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 287.486}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 310.894}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.215}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 248.388}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.552}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:51:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 323.631}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 288.612}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 281.007}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 322.148}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.361}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 270.356}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.92}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:52:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 329.834}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.372}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 315.794}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 305.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 277.005}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 261.191}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.035}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:53:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.12799072265625, "powerusage_W": 318.392}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.114}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 313.704}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 313.394}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.31}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 263.152}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.31}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:54:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 301.105}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.523}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 286.549}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 307.014}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.816}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 260.232}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.473}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:55:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.071}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.597}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 322.809}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.806}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.25}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 256.787}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.929}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:56:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.444}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.333}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 310.849}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 320.875}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 259.856}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.496}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:57:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 303.264}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 294.556}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 301.938}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.676}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 259.74}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.058}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:58:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.71783447265625, "powerusage_W": 295.375}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.647}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 280.971}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.487}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 134.997}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.079}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:59:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.53033447265625, "powerusage_W": 205.6}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 139.634}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.35369873046875, "powerusage_W": 336.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 137.788}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.81854248046875, "powerusage_W": 278.001}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 134.853}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.034}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:00:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.16119384765625, "powerusage_W": 198.94}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.38885498046875, "powerusage_W": 139.249}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.81658935546875, "powerusage_W": 314.017}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 45.1627197265625, "powerusage_W": 137.358}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 75.28143310546875, "powerusage_W": 273.34}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.5709228515625, "powerusage_W": 134.571}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.1021728515625, "powerusage_W": 135.732}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.998046875}]}
{"cur_time": "2024-08-26 05:01:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 293.373}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 343.638}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.01385498046875, "powerusage_W": 331.069}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 329.377}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.47674560546875, "powerusage_W": 313.21}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 320.918}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3267822265625, "powerusage_W": 333.035}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.22265625}]}
{"cur_time": "2024-08-26 05:02:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.30377197265625, "powerusage_W": 289.8}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 333.871}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.11151123046875, "powerusage_W": 343.32}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8150634765625, "powerusage_W": 320.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.47869873046875, "powerusage_W": 312.171}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 311.694}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 321.877}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:03:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 222.564}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 140.786}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.09393310546875, "powerusage_W": 296.307}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 138.052}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.56268310546875, "powerusage_W": 302.971}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 135.81}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6138916015625, "powerusage_W": 136.378}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.509765625}]}
{"cur_time": "2024-08-26 05:04:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.53033447265625, "powerusage_W": 204.315}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.35369873046875, "powerusage_W": 280.658}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 137.431}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.82244873046875, "powerusage_W": 273.621}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 135.653}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.039}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:05:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.65338134765625, "powerusage_W": 201.312}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.58807373046875, "powerusage_W": 140.373}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.93182373046875, "powerusage_W": 325.864}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.6549072265625, "powerusage_W": 137.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 73.40057373046875, "powerusage_W": 274.442}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.6861572265625, "powerusage_W": 135.711}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.2174072265625, "powerusage_W": 136.142}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 13.11328125}]}
{"cur_time": "2024-08-26 05:06:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 290.146}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 321.54}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.01385498046875, "powerusage_W": 320.879}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 320.485}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.48065185546875, "powerusage_W": 310.118}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 311.612}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3267822265625, "powerusage_W": 314.87}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.22265625}]}
{"cur_time": "2024-08-26 05:07:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.22760009765625, "powerusage_W": 305.696}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 141.492}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.03729248046875, "powerusage_W": 334.979}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 138.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.48260498046875, "powerusage_W": 261.276}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 136.618}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 137.572}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:08:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 219.345}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.195}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 329.25}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.3519287109375, "powerusage_W": 137.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 275.95}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.552}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.508}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:09:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.84674072265625, "powerusage_W": 198.517}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.86541748046875, "powerusage_W": 139.859}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 67.16815185546875, "powerusage_W": 291.335}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.0474853515625, "powerusage_W": 137.596}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.76776123046875, "powerusage_W": 269.052}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.0474853515625, "powerusage_W": 135.562}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.6627197265625, "powerusage_W": 135.315}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.55859375}]}
{"cur_time": "2024-08-26 05:10:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 296.992}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 140.517}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.90643310546875, "powerusage_W": 339.783}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8111572265625, "powerusage_W": 138.153}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.6881103515625, "powerusage_W": 132.165}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 136.288}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.2974853515625, "powerusage_W": 136.687}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.193359375}]}
{"cur_time": "2024-08-26 05:11:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.22760009765625, "powerusage_W": 296.481}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 146.492}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.03729248046875, "powerusage_W": 348.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 142.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.6900634765625, "powerusage_W": 135.137}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 138.476}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 141.262}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:12:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 205.622}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 145.138}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 293.224}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.815}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.074}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.218}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.969}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:13:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.57958984375, "powerusage_W": 143.707}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.715}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 341.945}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.705}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.991}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.454}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:14:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.65771484375, "powerusage_W": 352.01}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.774}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 337.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.407}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.919}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.781}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.709}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:15:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 345.692}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.773}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 329.886}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.382}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.796}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.625}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.475}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:16:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 335.762}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.361}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 308.23}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.726}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.426}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.142}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.053}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:17:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.84521484375, "powerusage_W": 334.038}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.687}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 305.197}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.811}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.913}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:18:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 348.407}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.593}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 291.587}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.21}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.712}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.461}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.635}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:19:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 339.992}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.379}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 309.034}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.93}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.481}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.371}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.42}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:20:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 147.13}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.569}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 332.456}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.777}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.367}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.286}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.303}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:21:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.87255859375, "powerusage_W": 142.14}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.456}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.69}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.517}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.314}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.226}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:22:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.89361572265625, "powerusage_W": 346.515}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.88690185546875, "powerusage_W": 375.662}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 324.634}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.1353759765625, "powerusage_W": 365.74}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.4967041015625, "powerusage_W": 367.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.2427978515625, "powerusage_W": 360.64}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:23:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.91705322265625, "powerusage_W": 354.116}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 381.866}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 308.342}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 370.12}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 365.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.7838134765625, "powerusage_W": 363.207}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.952}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:24:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.96002197265625, "powerusage_W": 369.53}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 381.09}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 299.485}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.753}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 367.08}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0474853515625, "powerusage_W": 360.675}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.521}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:25:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.46978759765625, "powerusage_W": 362.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 388.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 297.359}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 369.577}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 360.384}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0474853515625, "powerusage_W": 370.529}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.241}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:26:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.46978759765625, "powerusage_W": 357.594}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 385.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.906}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 375.82}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 359.126}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 359.493}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.226}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:27:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.74713134765625, "powerusage_W": 349.798}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 378.301}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 281.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.651}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 363.695}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 377.954}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.188}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:28:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.74713134765625, "powerusage_W": 360.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 385.96}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 325.173}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 378.9}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 369.781}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 361.498}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.151}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:29:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.77447509765625, "powerusage_W": 360.805}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 382.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 311.623}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.214}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 368.797}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 361.639}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.104}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:30:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 346.449}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 387.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.216}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 375.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 363.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 368.138}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.071}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:31:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 364.147}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 383.851}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 292.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 369.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 362.68}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 370.508}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.02}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:32:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 370.093}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 148.674}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.793}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.2877197265625, "powerusage_W": 384.687}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.7740478515625, "powerusage_W": 366.49}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 365.32}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 139.057}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:33:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 366.65}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 307.499}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.585}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.508}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.345}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:34:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 359.879}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.269}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.268}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.248}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.996}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.185}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.492}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:35:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 353.193}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.374}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 290.79}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.982}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.059}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.245}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.652}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:36:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.71783447265625, "powerusage_W": 387.943}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 284.587}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.758}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.654}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.755}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.269}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:37:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.27252197265625, "powerusage_W": 362.326}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 336.232}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.281}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.969}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:38:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.8502197265625, "powerusage_W": 347.998}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.893}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 321.0}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.273}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.015}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.73}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:39:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 357.978}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.634}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 306.785}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.105}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.811}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.905}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.633}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:40:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 345.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 293.819}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.371}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.607}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.586}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.277}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:41:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 363.674}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.389}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 313.263}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.853}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.403}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.56}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.294}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:42:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 342.667}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.001}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 304.662}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.656}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.121}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:43:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 341.943}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 290.737}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.72}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.737}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.784}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.167}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:44:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 22.43463134765625, "powerusage_W": 340.705}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 141.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.14471435546875, "powerusage_W": 338.6}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 137.601}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.2713623046875, "powerusage_W": 131.996}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.538}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:45:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 347.797}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.493}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 332.982}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.934}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.907}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.985}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:46:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 336.213}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.487}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 317.586}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.99}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.171}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.285}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.59}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:47:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.868}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.653}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.40447998046875, "powerusage_W": 310.332}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.285}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.317}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.482}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.669}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3591591, "create_time": 1724651217.99, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:48:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.236}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.587}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.24}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.304}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.337}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.368}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.745}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:49:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.70654296875, "powerusage_W": 140.968}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.48651123046875, "powerusage_W": 140.683}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.89862060546875, "powerusage_W": 281.568}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.4967041015625, "powerusage_W": 136.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.5220947265625, "powerusage_W": 131.599}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.8775634765625, "powerusage_W": 136.699}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.1334228515625, "powerusage_W": 136.995}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3592852, "create_time": 1724651301.87, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.029296875}]}
{"cur_time": "2024-08-26 05:50:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 7.50732421875, "powerusage_W": 140.42}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.01190185546875, "powerusage_W": 139.96}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.80487060546875, "powerusage_W": 301.592}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.8131103515625, "powerusage_W": 136.181}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.1138916015625, "powerusage_W": 130.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.1939697265625, "powerusage_W": 136.293}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.7252197265625, "powerusage_W": 136.372}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3594123, "create_time": 1724651385.9, "name": "pt_main_thread", "user": "user", "used_mem_GB": 5.62109375}]}
{"cur_time": "2024-08-26 05:51:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.15771484375, "powerusage_W": 139.758}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.562}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 279.07}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.615}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.501}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.794}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.884}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:52:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.38232421875, "powerusage_W": 139.597}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.05}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 326.272}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.657}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.54}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.82}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:53:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.685}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.891}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 298.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.781}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.482}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.956}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:54:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.51708984375, "powerusage_W": 141.489}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.14666748046875, "powerusage_W": 141.029}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 65.62518310546875, "powerusage_W": 290.889}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.4967041015625, "powerusage_W": 137.325}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.2486572265625, "powerusage_W": 131.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.4967041015625, "powerusage_W": 136.924}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.8599853515625, "powerusage_W": 137.082}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.755859375}]}
{"cur_time": "2024-08-26 05:55:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.68896484375, "powerusage_W": 143.559}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 143.52}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 76.75799560546875, "powerusage_W": 339.56}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5845947265625, "powerusage_W": 139.635}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 132.716}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 137.541}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1158447265625, "powerusage_W": 138.791}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.01171875}]}
{"cur_time": "2024-08-26 05:56:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 143.624}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 144.523}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 348.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 139.493}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 134.199}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 137.737}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 139.251}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 05:57:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.62646484375, "powerusage_W": 141.472}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78143310546875, "powerusage_W": 142.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.25994873046875, "powerusage_W": 351.051}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.9635009765625, "powerusage_W": 138.144}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.9888916015625, "powerusage_W": 131.951}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 136.224}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 137.382}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3599799, "create_time": 1724651831.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-26 05:58:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.57958984375, "powerusage_W": 141.151}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19940185546875, "powerusage_W": 141.702}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 67.50994873046875, "powerusage_W": 321.731}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.2135009765625, "powerusage_W": 138.208}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.2174072265625, "powerusage_W": 131.378}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.3814697265625, "powerusage_W": 136.263}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7447509765625, "powerusage_W": 136.875}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.640625}]}
{"cur_time": "2024-08-26 05:59:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 313.445}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 320.27}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 335.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 309.965}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 293.277}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 309.245}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 308.218}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:00:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 142.483}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 143.063}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 355.718}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 138.572}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 132.856}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 136.362}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 138.073}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:01:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.214}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 141.626}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.14471435546875, "powerusage_W": 302.053}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 137.957}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7681884765625, "powerusage_W": 131.548}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 136.452}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6138916015625, "powerusage_W": 137.342}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.509765625}]}
{"cur_time": "2024-08-26 06:02:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.87841796875, "powerusage_W": 140.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.34002685546875, "powerusage_W": 141.185}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 72.67205810546875, "powerusage_W": 291.237}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.7740478515625, "powerusage_W": 137.644}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6939697265625, "powerusage_W": 131.042}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.1451416015625, "powerusage_W": 136.213}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.3052978515625, "powerusage_W": 136.526}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.201171875}]}
{"cur_time": "2024-08-26 06:03:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 326.135}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 311.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 72.79119873046875, "powerusage_W": 317.325}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 312.02}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 306.301}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 136.835}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 290.791}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:04:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.689}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.899}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 308.788}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.789}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.676}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.429}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.225}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:05:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.48388671875, "powerusage_W": 236.575}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.427}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 320.467}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.592}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.665}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.628}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.975}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:06:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47802734375, "powerusage_W": 346.426}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.692}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 290.416}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.689}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.495}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.591}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.899}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:07:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47802734375, "powerusage_W": 343.48}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.444}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 296.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.383}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.482}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.514}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.923}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:08:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47998046875, "powerusage_W": 341.683}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 294.092}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.478}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.34}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.457}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.822}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:09:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47998046875, "powerusage_W": 339.951}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.181}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.09783935546875, "powerusage_W": 289.789}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.266}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.173}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.302}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.751}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:10:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.63970947265625, "powerusage_W": 295.26}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.41619873046875, "powerusage_W": 280.204}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 387.697}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.9420166015625, "powerusage_W": 328.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 278.847}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.0299072265625, "powerusage_W": 247.705}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.443}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:11:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.63970947265625, "powerusage_W": 300.721}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 281.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 343.606}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.0494384765625, "powerusage_W": 318.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 281.003}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.0299072265625, "powerusage_W": 249.081}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.427}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:12:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04400634765625, "powerusage_W": 298.248}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 289.231}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 328.141}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2603759765625, "powerusage_W": 312.913}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 282.709}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 255.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.418}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:13:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.04833984375, "powerusage_W": 204.057}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 268.249}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 426.005}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2720947265625, "powerusage_W": 307.349}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 277.829}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 252.688}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.596}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:14:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.55963134765625, "powerusage_W": 295.794}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.141}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 346.662}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2838134765625, "powerusage_W": 310.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 276.991}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 248.042}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.635}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:15:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04205322265625, "powerusage_W": 309.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.119}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 406.556}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3970947265625, "powerusage_W": 304.214}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.457}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.2545166015625, "powerusage_W": 248.764}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.551}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:16:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04205322265625, "powerusage_W": 290.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 278.664}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 332.399}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3970947265625, "powerusage_W": 310.685}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 274.854}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.265}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.873}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:17:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45416259765625, "powerusage_W": 314.019}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 291.727}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 353.897}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 328.048}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.313}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 254.51}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.61}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:18:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 296.616}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 289.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 347.718}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 323.908}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 276.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 250.95}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.471}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:19:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 293.781}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 292.25}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 342.762}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 323.265}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 274.732}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.51}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:20:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 315.16}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 290.287}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 336.096}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 319.765}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 273.555}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 264.457}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.656}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:21:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 316.788}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 291.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 331.798}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 319.907}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 279.855}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 248.352}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.323}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:22:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 298.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 282.371}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 398.965}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 316.996}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 272.595}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 248.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.107}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:23:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.5982666015625, "powerusage_W": 315.832}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 54.068115234375, "powerusage_W": 284.261}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 89.450927734375, "powerusage_W": 420.678}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15057373046875, "powerusage_W": 315.424}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 64.67791748046875, "powerusage_W": 277.756}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.00408935546875, "powerusage_W": 246.631}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8287353515625, "powerusage_W": 136.97}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3616690, "create_time": 1724653383.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.724609375}]}
{"cur_time": "2024-08-26 06:24:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.5982666015625, "powerusage_W": 298.57}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 54.068115234375, "powerusage_W": 288.962}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 89.450927734375, "powerusage_W": 344.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15057373046875, "powerusage_W": 311.452}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 64.67791748046875, "powerusage_W": 291.469}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.00408935546875, "powerusage_W": 247.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8287353515625, "powerusage_W": 136.57}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3617351, "create_time": 1724653450.15, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.724609375}]}
{"cur_time": "2024-08-26 06:25:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4713134765625, "powerusage_W": 285.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.909912109375, "powerusage_W": 291.257}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.292724609375, "powerusage_W": 336.679}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.99237060546875, "powerusage_W": 319.649}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.51971435546875, "powerusage_W": 277.728}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.84588623046875, "powerusage_W": 248.545}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3853759765625, "powerusage_W": 136.633}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3617351, "create_time": 1724653450.15, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.279296875}]}
{"cur_time": "2024-08-26 06:26:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.46044921875, "powerusage_W": 223.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 282.986}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 408.706}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 321.258}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 276.188}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 247.182}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.409}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:27:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.83502197265625, "powerusage_W": 285.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 279.326}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 402.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 318.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.214}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.257}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.171}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:28:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.6138916015625, "powerusage_W": 291.112}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.611083984375, "powerusage_W": 279.14}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 93.993896484375, "powerusage_W": 391.64}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.69354248046875, "powerusage_W": 311.88}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.22088623046875, "powerusage_W": 272.858}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.54705810546875, "powerusage_W": 249.828}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.4693603515625, "powerusage_W": 136.259}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 7.36328125}]}
{"cur_time": "2024-08-26 06:29:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4263916015625, "powerusage_W": 296.042}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.890380859375, "powerusage_W": 281.25}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.273193359375, "powerusage_W": 328.738}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.97283935546875, "powerusage_W": 311.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.50018310546875, "powerusage_W": 282.81}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.82635498046875, "powerusage_W": 248.366}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3736572265625, "powerusage_W": 136.001}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.267578125}]}
{"cur_time": "2024-08-26 06:30:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4263916015625, "powerusage_W": 292.394}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.890380859375, "powerusage_W": 286.352}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.273193359375, "powerusage_W": 346.719}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.97283935546875, "powerusage_W": 317.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.50213623046875, "powerusage_W": 276.582}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.95135498046875, "powerusage_W": 258.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3736572265625, "powerusage_W": 135.99}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.267578125}]}
{"cur_time": "2024-08-26 06:31:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45416259765625, "powerusage_W": 297.031}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 285.718}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 346.254}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 310.428}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 275.238}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 248.741}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.944}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:32:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 323.825}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 278.639}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 409.21}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 312.648}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 273.979}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 255.409}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.002}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:33:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 295.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 279.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 353.471}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 308.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.654}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 245.291}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.788}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:34:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 297.6}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.32}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 340.845}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 308.419}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 279.155}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 246.301}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.788}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:35:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 291.868}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 287.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 329.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 320.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.673}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 248.512}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.524}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:36:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 290.544}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 284.997}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 336.716}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 313.983}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.421}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 261.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.544}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:37:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 296.35}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 283.42}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 343.146}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 310.729}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.024}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 245.88}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.52}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:38:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 290.45}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 281.809}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 345.674}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 311.361}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.891}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 253.455}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.425}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:39:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 301.222}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.329}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 342.566}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 309.825}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.806}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 261.929}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.591}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:40:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.88623046875, "powerusage_W": 237.753}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 276.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 344.464}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 314.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 259.566}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 239.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.403}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:41:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.88623046875, "powerusage_W": 198.72}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 285.704}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 394.92}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 311.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 268.271}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 250.608}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.342}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
Reference in New Issue View Git Blame Copy Permalink