LLaMA-Factory-Mirror/results/gpu_status/gpu_status_0826_2.json

35 lines
41 KiB
JSON

{"cur_time": "2024-08-26 07:05:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.725}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.821}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 387.835}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.1}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 129.631}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 134.659}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.641}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:06:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 7.76904296875, "powerusage_W": 139.406}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.48260498046875, "powerusage_W": 139.795}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 335.86}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.947}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.5845947265625, "powerusage_W": 131.183}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.6646728515625, "powerusage_W": 134.525}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.1959228515625, "powerusage_W": 135.534}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3643236, "create_time": 1724655972.4, "name": "pt_main_thread", "user": "user", "used_mem_GB": 6.091796875}]}
{"cur_time": "2024-08-26 07:07:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.505}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.692}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.787}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.22}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 134.768}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.686}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:11:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 138.749}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 137.602}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.43}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.72}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.889}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 134.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.388}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:12:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.39794921875, "powerusage_W": 142.378}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.73065185546875, "powerusage_W": 140.706}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.337}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.476}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.8990478515625, "powerusage_W": 133.656}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.1881103515625, "powerusage_W": 137.01}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.8248291015625, "powerusage_W": 138.074}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3647761, "create_time": 1724656298.62, "name": "pt_main_thread", "user": "user", "used_mem_GB": 7.720703125}]}
{"cur_time": "2024-08-26 07:13:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.71240234375, "powerusage_W": 143.84}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.42596435546875, "powerusage_W": 141.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.385}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.691}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.5279541015625, "powerusage_W": 136.37}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6080322265625, "powerusage_W": 138.713}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.1392822265625, "powerusage_W": 138.233}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3649322, "create_time": 1724656386.83, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.03515625}]}
{"cur_time": "2024-08-26 07:14:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.945}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.651}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.268}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.543}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 135.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.46}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.394}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:15:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 14.03466796875, "powerusage_W": 142.033}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.58612060546875, "powerusage_W": 140.732}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.399}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.697}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.8502197265625, "powerusage_W": 134.01}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.1392822265625, "powerusage_W": 139.412}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.4302978515625, "powerusage_W": 137.571}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3650694, "create_time": 1724656464.9, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.326171875}]}
{"cur_time": "2024-08-26 07:16:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.116}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.747}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.35}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.616}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.1}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.321}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.44}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:17:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.653}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.324}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.964}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.156}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.665}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.036}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.0767822265625, "powerusage_W": 137.162}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3653417, "create_time": 1724656631.3, "name": "pt_main_thread", "user": "user", "used_mem_GB": 2.97265625}]}
{"cur_time": "2024-08-26 07:18:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.66748046875, "powerusage_W": 141.636}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.13299560546875, "powerusage_W": 140.342}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.81}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.114}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.425}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.797}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.935}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:19:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.68115234375, "powerusage_W": 377.912}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.93768310546875, "powerusage_W": 389.617}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.398}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.2877197265625, "powerusage_W": 359.813}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.7349853515625, "powerusage_W": 372.639}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.507}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.503}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:20:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.68115234375, "powerusage_W": 382.973}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.93768310546875, "powerusage_W": 379.247}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.346}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.6978759765625, "powerusage_W": 369.157}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.0592041015625, "powerusage_W": 378.25}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.6685791015625, "powerusage_W": 367.009}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.393}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:21:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.95068359375, "powerusage_W": 387.557}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.93768310546875, "powerusage_W": 384.584}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.207}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.6978759765625, "powerusage_W": 365.235}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.0592041015625, "powerusage_W": 368.217}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.1373291015625, "powerusage_W": 367.037}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.215}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:22:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.95068359375, "powerusage_W": 381.969}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.93768310546875, "powerusage_W": 381.494}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.041}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 364.031}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.0592041015625, "powerusage_W": 376.895}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.4010009765625, "powerusage_W": 368.992}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.104}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:23:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.95068359375, "powerusage_W": 387.594}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.93768310546875, "powerusage_W": 373.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.89}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 372.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.2642822265625, "powerusage_W": 373.365}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.4010009765625, "powerusage_W": 365.793}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.984}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:24:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.95068359375, "powerusage_W": 385.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.450927734375, "powerusage_W": 377.046}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.854}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 371.539}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.77752685546875, "powerusage_W": 359.723}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.91619873046875, "powerusage_W": 366.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6099853515625, "powerusage_W": 136.179}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3658612, "create_time": 1724657060.62, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.505859375}]}
{"cur_time": "2024-08-26 07:25:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.73736572265625, "powerusage_W": 251.825}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.212646484375, "powerusage_W": 388.295}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.793}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 373.825}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 66.54119873046875, "powerusage_W": 224.627}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.67791748046875, "powerusage_W": 228.891}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.1392822265625, "powerusage_W": 136.35}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3658612, "create_time": 1724657060.62, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.03515625}]}
{"cur_time": "2024-08-26 07:26:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.73736572265625, "powerusage_W": 235.199}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.212646484375, "powerusage_W": 375.606}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.715}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 366.58}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 66.54119873046875, "powerusage_W": 233.397}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.67791748046875, "powerusage_W": 227.198}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.1392822265625, "powerusage_W": 136.297}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3658612, "create_time": 1724657060.62, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.03515625}]}
{"cur_time": "2024-08-26 07:27:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.64556884765625, "powerusage_W": 347.235}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 74.126708984375, "powerusage_W": 326.092}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.674}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 374.83}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.48455810546875, "powerusage_W": 238.612}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 74.62127685546875, "powerusage_W": 332.901}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.0650634765625, "powerusage_W": 136.166}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3658612, "create_time": 1724657060.62, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.9609375}]}
{"cur_time": "2024-08-26 07:28:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.50927734375, "powerusage_W": 371.801}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.450927734375, "powerusage_W": 376.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.57}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 364.267}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.77947998046875, "powerusage_W": 364.618}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.91619873046875, "powerusage_W": 363.687}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 135.205}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3661247, "create_time": 1724657298.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-26 07:29:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.73736572265625, "powerusage_W": 233.467}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.212646484375, "powerusage_W": 386.575}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.301}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.9478759765625, "powerusage_W": 365.793}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 66.54119873046875, "powerusage_W": 220.913}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.67791748046875, "powerusage_W": 246.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.1392822265625, "powerusage_W": 135.445}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3661247, "create_time": 1724657298.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.03515625}]}
{"cur_time": "2024-08-26 07:30:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.68072509765625, "powerusage_W": 383.877}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.30291748046875, "powerusage_W": 141.554}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.367}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 67.45330810546875, "powerusage_W": 371.692}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.59002685546875, "powerusage_W": 381.54}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0513916015625, "powerusage_W": 137.296}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3661247, "create_time": 1724657298.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.947265625}]}
{"cur_time": "2024-08-26 07:31:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.66314697265625, "powerusage_W": 349.397}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.35369873046875, "powerusage_W": 157.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.543}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.148}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.48455810546875, "powerusage_W": 244.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 74.62127685546875, "powerusage_W": 249.556}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.0650634765625, "powerusage_W": 160.572}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3661247, "create_time": 1724657298.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.9609375}]}
{"cur_time": "2024-08-26 07:32:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.64208984375, "powerusage_W": 143.574}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.35369873046875, "powerusage_W": 140.566}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.731}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.128}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 66.47283935546875, "powerusage_W": 369.899}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.35760498046875, "powerusage_W": 363.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.0709228515625, "powerusage_W": 137.719}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3663542, "create_time": 1724657531.02, "name": "pt_main_thread", "user": "user", "used_mem_GB": 2.966796875}]}
{"cur_time": "2024-08-26 07:33:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.62255859375, "powerusage_W": 285.573}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.34002685546875, "powerusage_W": 298.656}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.901}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.361}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.4732666015625, "powerusage_W": 177.819}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 74.62127685546875, "powerusage_W": 344.482}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.0650634765625, "powerusage_W": 276.468}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3663542, "create_time": 1724657531.02, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.9609375}]}
{"cur_time": "2024-08-26 07:34:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.581}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.986}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.932}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.652}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.905}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.6939697265625, "powerusage_W": 137.015}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3664716, "create_time": 1724657632.19, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.58984375}]}
{"cur_time": "2024-08-26 07:35:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.559}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.065}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.178}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.67}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.7252197265625, "powerusage_W": 291.986}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3664716, "create_time": 1724657632.19, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.62109375}]}
{"cur_time": "2024-08-26 07:36:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.881}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.347}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.23}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.743}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.952}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.721}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.9615478515625, "powerusage_W": 136.581}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3666292, "create_time": 1724657759.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 7.857421875}]}
{"cur_time": "2024-08-26 07:37:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.99}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.608}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.236}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.599}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.693}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.892}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.5689697265625, "powerusage_W": 300.83}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3666292, "create_time": 1724657759.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.46484375}]}
{"cur_time": "2024-08-26 07:38:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.971}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.59}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.024}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.352}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.03}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7720947265625, "powerusage_W": 136.72}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3667726, "create_time": 1724657889.11, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.66796875}]}
{"cur_time": "2024-08-26 07:39:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.62}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.266}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.97}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.363}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.421}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.772}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.5689697265625, "powerusage_W": 282.361}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3667726, "create_time": 1724657889.11, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.46484375}]}
{"cur_time": "2024-08-26 07:40:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.476}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.914}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.322}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.242}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.545}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.196}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 07:41:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 139.134}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 137.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.606}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.59}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.34}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.606}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.434}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}