p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity
LLaMA-Factory-Mirror/results/gpu_status.json

976 lines
743 KiB
JSON
Raw Blame History

{"cur_time": "2024-08-20 05:51:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 307.167}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 422.759}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 277.214}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 338.271}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 336.763}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 242.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 263.752}]}
{"cur_time": "2024-08-20 05:51:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 346.189}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 429.544}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 327.636}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 291.271}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 341.014}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 293.648}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 318.306}]}
{"cur_time": "2024-08-20 05:51:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 379.941}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 307.755}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 376.886}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 290.816}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 258.29}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 273.436}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 425.757}]}
{"cur_time": "2024-08-20 05:51:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 308.645}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 407.742}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 338.193}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 317.69}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 349.583}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 351.25}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 408.942}]}
{"cur_time": "2024-08-20 05:51:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 423.708}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 330.528}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 335.454}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 335.813}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 302.27}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 276.465}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 425.131}]}
{"cur_time": "2024-08-20 05:52:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 347.216}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 336.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19329833984375, "powerusage_W": 369.65}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 375.804}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 249.434}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 375.997}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 336.457}]}
{"cur_time": "2024-08-20 05:52:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 363.441}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 401.156}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 357.961}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 265.645}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 286.166}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 331.34}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 356.879}]}
{"cur_time": "2024-08-20 05:52:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 264.631}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 302.68}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 454.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 289.179}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 261.585}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 326.945}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 372.993}]}
{"cur_time": "2024-08-20 05:52:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 337.774}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 345.174}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 335.228}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 286.173}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18548583984375, "powerusage_W": 287.199}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 283.311}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 422.488}]}
{"cur_time": "2024-08-20 05:52:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 462.293}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 373.353}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 321.738}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 282.028}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 333.428}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 406.635}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 305.511}]}
{"cur_time": "2024-08-20 05:52:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 384.12}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 335.057}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 445.323}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 320.908}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 355.983}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 361.616}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 301.339}]}
{"cur_time": "2024-08-20 05:52:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 366.053}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 269.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 377.02}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 343.33}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 316.924}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 324.999}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 403.598}]}
{"cur_time": "2024-08-20 05:52:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 336.677}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 344.461}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 409.826}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 280.51}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 381.218}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 352.569}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 324.622}]}
{"cur_time": "2024-08-20 05:52:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 343.702}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 326.647}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 407.118}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 374.119}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 355.588}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 267.23}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97259521484375, "powerusage_W": 273.832}]}
{"cur_time": "2024-08-20 05:52:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 298.007}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 352.17}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19525146484375, "powerusage_W": 329.054}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 319.866}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 323.981}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 388.235}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 264.466}]}
{"cur_time": "2024-08-20 05:52:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 353.069}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 342.228}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 385.345}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 318.174}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 361.891}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 318.237}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 321.576}]}
{"cur_time": "2024-08-20 05:52:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 379.997}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 360.658}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 310.646}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 348.472}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 390.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 332.858}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 380.053}]}
{"cur_time": "2024-08-20 05:53:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 301.181}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 299.387}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 378.149}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 317.838}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 250.537}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 376.788}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 311.201}]}
{"cur_time": "2024-08-20 05:53:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 327.049}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 353.297}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 312.501}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 373.248}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 367.924}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 382.531}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 300.593}]}
{"cur_time": "2024-08-20 05:53:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 391.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 397.56}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 302.363}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 375.895}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 302.32}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 346.986}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 307.599}]}
{"cur_time": "2024-08-20 05:53:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 385.476}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 365.405}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 338.363}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 304.719}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 343.358}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 371.967}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 294.259}]}
{"cur_time": "2024-08-20 05:53:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 296.326}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 282.231}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 287.786}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 353.057}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 362.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 416.269}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 441.139}]}
{"cur_time": "2024-08-20 05:53:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 341.974}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 368.386}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 313.315}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 343.119}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 285.099}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 358.607}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 338.021}]}
{"cur_time": "2024-08-20 05:53:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 373.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 390.075}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 380.943}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 338.585}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 345.427}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 311.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 339.123}]}
{"cur_time": "2024-08-20 05:53:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 271.227}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 402.251}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 295.655}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 271.54}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 330.76}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 311.469}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 329.33}]}
{"cur_time": "2024-08-20 05:53:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 333.173}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 364.767}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 320.433}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 334.655}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 292.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 392.604}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 364.216}]}
{"cur_time": "2024-08-20 05:53:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 324.805}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 353.805}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 386.361}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 350.295}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 336.418}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 337.784}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 285.147}]}
{"cur_time": "2024-08-20 05:53:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 348.019}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 381.006}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 292.084}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 378.336}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 306.725}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 313.626}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 301.81}]}
{"cur_time": "2024-08-20 05:53:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 302.408}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 312.465}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 387.608}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 355.132}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 308.182}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 384.889}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 368.923}]}
{"cur_time": "2024-08-20 05:54:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 273.023}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 363.172}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 302.859}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 345.856}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 319.584}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 351.944}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 259.696}]}
{"cur_time": "2024-08-20 05:54:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 324.606}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 351.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 364.526}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 328.148}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 339.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 400.083}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 266.148}]}
{"cur_time": "2024-08-20 05:54:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.14642333984375, "powerusage_W": 332.903}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 384.642}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.19720458984375, "powerusage_W": 297.204}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 315.529}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 319.422}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18743896484375, "powerusage_W": 407.878}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.97454833984375, "powerusage_W": 367.279}]}
{"cur_time": "2024-08-20 05:54:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.5882568359375, "powerusage_W": 96.803}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.5882568359375, "powerusage_W": 132.721}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.5882568359375, "powerusage_W": 142.463}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.8695068359375, "powerusage_W": 132.029}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.18939208984375, "powerusage_W": 132.193}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.8695068359375, "powerusage_W": 139.223}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.5882568359375, "powerusage_W": 130.807}]}
{"cur_time": "2024-08-20 05:54:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.073}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 90.727}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.418}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.977}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.475}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.583}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.108}]}
{"cur_time": "2024-08-20 05:54:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.58}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 90.199}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.173}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.671}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.091}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.351}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.854}]}
{"cur_time": "2024-08-20 05:54:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.774}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.025}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.39}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.804}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.268}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.775}]}
{"cur_time": "2024-08-20 05:54:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.85}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.503}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.85}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.257}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.531}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.579}]}
{"cur_time": "2024-08-20 05:54:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.662}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.253}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.765}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.178}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.353}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.994}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.514}]}
{"cur_time": "2024-08-20 05:54:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.461}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.096}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.762}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.215}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.216}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.896}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.302}]}
{"cur_time": "2024-08-20 05:54:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.454}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.061}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.933}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.302}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.924}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.403}]}
{"cur_time": "2024-08-20 05:54:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.476}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.042}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.94}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.494}]}
{"cur_time": "2024-08-20 05:55:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.606}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.173}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.563}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.198}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.054}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.605}]}
{"cur_time": "2024-08-20 05:55:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.707}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.223}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.233}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.641}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.24}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.726}]}
{"cur_time": "2024-08-20 05:55:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.749}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.282}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.408}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.832}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.356}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.797}]}
{"cur_time": "2024-08-20 05:55:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.837}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.383}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.543}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.961}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.468}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.18}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.924}]}
{"cur_time": "2024-08-20 05:55:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.009}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.535}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.621}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.066}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.539}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.316}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.971}]}
{"cur_time": "2024-08-20 06:03:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.105}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.662}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.544}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.076}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.668}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.385}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.917}]}
{"cur_time": "2024-08-20 06:03:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.086}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.609}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.47}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.968}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.603}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.373}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.86}]}
{"cur_time": "2024-08-20 06:03:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.047}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.557}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.434}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.979}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.584}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.323}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.841}]}
{"cur_time": "2024-08-20 06:03:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.047}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.547}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.377}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.943}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.59}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.324}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.842}]}
{"cur_time": "2024-08-20 06:04:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.001}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.506}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.413}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.966}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.54}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.257}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.836}]}
{"cur_time": "2024-08-20 06:04:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.021}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.486}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 95.345}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.986}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.52}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.311}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.823}]}
{"cur_time": "2024-08-20 09:42:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.812}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.565}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.019}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.997}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.677}]}
{"cur_time": "2024-08-20 09:43:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 92.277}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 133.747}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 137.804}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 126.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 126.724}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 134.597}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 130.021}]}
{"cur_time": "2024-08-20 09:44:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.31634521484375, "powerusage_W": 138.029}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 133.229}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 137.83}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 126.464}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 126.853}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.604}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.12884521484375, "powerusage_W": 130.027}]}
{"cur_time": "2024-08-20 09:45:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.85150146484375, "powerusage_W": 137.919}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.52923583984375, "powerusage_W": 133.329}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.27728271484375, "powerusage_W": 137.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.27728271484375, "powerusage_W": 127.176}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.91400146484375, "powerusage_W": 126.835}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.36126708984375, "powerusage_W": 134.646}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.04290771484375, "powerusage_W": 129.952}]}
{"cur_time": "2024-08-20 09:46:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21478271484375, "powerusage_W": 263.441}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.00579833984375, "powerusage_W": 285.782}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.26361083984375, "powerusage_W": 313.878}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 310.707}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 413.136}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29486083984375, "powerusage_W": 302.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.05462646484375, "powerusage_W": 330.241}]}
{"cur_time": "2024-08-20 09:47:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21478271484375, "powerusage_W": 305.903}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01165771484375, "powerusage_W": 293.672}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 245.825}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 293.173}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 273.819}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 408.555}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 230.08}]}
{"cur_time": "2024-08-20 09:48:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25189208984375, "powerusage_W": 334.672}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01165771484375, "powerusage_W": 397.117}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 304.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 320.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 252.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 269.689}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 327.85}]}
{"cur_time": "2024-08-20 09:49:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25189208984375, "powerusage_W": 257.108}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 315.719}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 355.586}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 232.663}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 302.526}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 427.71}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 243.456}]}
{"cur_time": "2024-08-20 09:50:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25189208984375, "powerusage_W": 287.038}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 267.942}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 318.297}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 273.207}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 339.951}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 245.382}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 297.62}]}
{"cur_time": "2024-08-20 09:51:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25384521484375, "powerusage_W": 301.718}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 368.199}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 302.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 231.851}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 277.293}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 332.93}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 355.917}]}
{"cur_time": "2024-08-20 09:52:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25384521484375, "powerusage_W": 354.449}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 393.031}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 320.834}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 320.288}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 357.31}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 297.024}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06439208984375, "powerusage_W": 371.189}]}
{"cur_time": "2024-08-20 09:53:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25384521484375, "powerusage_W": 311.967}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 257.526}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 384.156}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 327.262}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 270.265}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 343.001}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 291.153}]}
{"cur_time": "2024-08-20 09:54:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 342.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 365.776}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 318.841}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 341.386}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 299.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 274.17}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 304.046}]}
{"cur_time": "2024-08-20 09:55:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 320.066}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 299.799}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 329.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 245.318}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 247.275}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 385.724}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 356.631}]}
{"cur_time": "2024-08-20 09:56:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 292.478}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 341.461}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 274.637}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 278.702}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 400.695}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 256.45}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 264.054}]}
{"cur_time": "2024-08-20 09:57:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 285.642}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 251.074}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29876708984375, "powerusage_W": 245.605}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 425.414}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 363.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 265.746}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 293.733}]}
{"cur_time": "2024-08-20 09:58:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 318.925}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 346.163}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 306.465}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 407.933}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 326.784}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 295.638}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 304.15}]}
{"cur_time": "2024-08-20 09:59:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 294.651}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 313.803}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 443.179}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 264.639}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 293.404}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 302.704}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 249.412}]}
{"cur_time": "2024-08-20 10:00:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 329.66}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 280.313}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 309.254}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 304.553}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 382.274}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 349.281}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 270.299}]}
{"cur_time": "2024-08-20 10:01:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 372.666}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 245.213}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 304.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 259.377}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 290.017}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 259.432}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 375.431}]}
{"cur_time": "2024-08-20 10:02:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 262.391}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 311.891}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 339.171}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 300.014}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 267.445}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 389.603}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 309.464}]}
{"cur_time": "2024-08-20 10:03:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 240.942}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 289.733}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 296.249}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 430.667}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 219.067}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 323.875}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 233.058}]}
{"cur_time": "2024-08-20 10:04:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 329.173}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 328.881}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 258.33}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 260.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 283.311}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 360.496}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 391.37}]}
{"cur_time": "2024-08-20 10:05:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 443.029}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 258.569}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 259.868}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 274.47}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 313.017}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 297.171}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 356.32}]}
{"cur_time": "2024-08-20 10:06:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 372.111}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 296.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 353.957}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 258.428}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 350.545}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 316.202}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 244.996}]}
{"cur_time": "2024-08-20 10:07:28", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 375.767}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 269.283}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 346.143}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 263.675}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 286.4}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 321.868}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 286.688}]}
{"cur_time": "2024-08-20 10:08:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 280.367}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 361.332}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 262.573}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 233.42}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 349.636}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 327.772}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 253.158}]}
{"cur_time": "2024-08-20 10:09:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 353.833}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 338.383}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 258.09}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 231.466}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 275.415}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 359.419}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 292.84}]}
{"cur_time": "2024-08-20 10:10:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 372.537}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 341.905}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 223.703}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 288.734}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 249.12}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 298.308}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 298.282}]}
{"cur_time": "2024-08-20 10:11:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 290.978}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 295.673}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 319.671}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 325.035}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 287.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 360.708}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 307.219}]}
{"cur_time": "2024-08-20 10:12:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 240.246}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 284.353}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 285.777}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 240.162}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 296.369}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 334.66}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 244.934}]}
{"cur_time": "2024-08-20 10:13:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 287.42}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 273.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 305.716}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 272.618}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 335.146}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 321.721}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 356.518}]}
{"cur_time": "2024-08-20 10:14:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 275.991}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 307.55}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 250.223}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 353.683}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 279.027}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 324.032}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 338.185}]}
{"cur_time": "2024-08-20 10:15:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 354.323}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 341.479}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 359.89}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 326.511}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 227.855}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 271.513}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 287.663}]}
{"cur_time": "2024-08-20 10:16:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 299.16}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 406.268}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 266.89}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 324.074}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 324.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 346.194}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 293.205}]}
{"cur_time": "2024-08-20 10:17:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 315.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 318.921}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 271.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 391.682}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 289.021}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 348.939}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 281.629}]}
{"cur_time": "2024-08-20 10:18:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 297.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 340.067}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 375.461}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 312.612}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 359.577}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 297.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 287.699}]}
{"cur_time": "2024-08-20 10:19:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 263.689}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 297.215}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 301.385}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 344.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 417.172}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 272.395}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 204.118}]}
{"cur_time": "2024-08-20 10:20:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 280.548}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 457.542}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 344.048}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 312.8}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 234.011}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 340.347}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 287.453}]}
{"cur_time": "2024-08-20 10:21:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 256.375}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 323.462}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 331.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 317.722}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 249.961}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 323.57}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 285.287}]}
{"cur_time": "2024-08-20 10:22:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 275.957}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 434.18}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 270.797}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 292.912}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 293.395}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 366.161}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 262.699}]}
{"cur_time": "2024-08-20 10:23:29", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.25579833984375, "powerusage_W": 391.384}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.01751708984375, "powerusage_W": 225.071}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30267333984375, "powerusage_W": 335.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 255.342}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 324.061}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 288.833}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06634521484375, "powerusage_W": 231.678}]}
{"cur_time": "2024-08-20 10:24:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.046}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 135.751}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 139.581}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 128.338}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 129.562}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 135.549}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 130.885}]}
{"cur_time": "2024-08-20 10:25:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.31634521484375, "powerusage_W": 139.227}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.573}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 139.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 129.237}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 128.465}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.79}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.12884521484375, "powerusage_W": 130.159}]}
{"cur_time": "2024-08-20 10:26:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.38275146484375, "powerusage_W": 138.617}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.85345458984375, "powerusage_W": 134.197}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.47650146484375, "powerusage_W": 138.894}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.75970458984375, "powerusage_W": 129.027}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.75970458984375, "powerusage_W": 127.954}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.67572021484375, "powerusage_W": 134.439}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.52532958984375, "powerusage_W": 129.832}]}
{"cur_time": "2024-08-20 10:27:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.95111083984375, "powerusage_W": 302.325}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.54486083984375, "powerusage_W": 345.845}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.29486083984375, "powerusage_W": 266.935}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 409.167}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31048583984375, "powerusage_W": 205.367}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.02923583984375, "powerusage_W": 249.029}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.07025146484375, "powerusage_W": 351.545}]}
{"cur_time": "2024-08-20 10:28:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.95111083984375, "powerusage_W": 317.113}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.57806396484375, "powerusage_W": 276.427}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 289.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 309.401}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 273.705}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 265.659}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 389.605}]}
{"cur_time": "2024-08-20 10:29:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.98822021484375, "powerusage_W": 311.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.57806396484375, "powerusage_W": 372.19}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 313.258}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 245.154}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 296.348}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 343.648}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 294.483}]}
{"cur_time": "2024-08-20 10:30:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.98822021484375, "powerusage_W": 242.641}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 274.21}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 240.638}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 278.865}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 291.807}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 344.354}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 254.073}]}
{"cur_time": "2024-08-20 10:31:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.98822021484375, "powerusage_W": 369.857}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 246.629}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 348.389}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 280.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 294.911}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 284.401}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 274.187}]}
{"cur_time": "2024-08-20 10:32:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99017333984375, "powerusage_W": 271.127}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 282.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 300.994}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 354.314}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 271.612}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 454.889}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 322.131}]}
{"cur_time": "2024-08-20 10:33:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99017333984375, "powerusage_W": 366.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 350.142}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 280.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 361.257}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 309.528}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 311.54}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10736083984375, "powerusage_W": 225.785}]}
{"cur_time": "2024-08-20 10:34:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99017333984375, "powerusage_W": 312.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 282.961}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 233.264}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 321.865}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 223.17}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 395.198}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 218.37}]}
{"cur_time": "2024-08-20 10:35:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 327.597}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 364.339}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 371.728}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 382.204}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 264.857}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 265.737}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 205.608}]}
{"cur_time": "2024-08-20 10:36:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 251.582}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 283.317}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 299.383}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 387.2}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 393.691}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 299.496}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 286.78}]}
{"cur_time": "2024-08-20 10:37:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 333.816}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 301.247}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 302.512}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 310.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 305.511}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 281.0}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 337.687}]}
{"cur_time": "2024-08-20 10:38:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 262.499}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 357.652}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33001708984375, "powerusage_W": 267.086}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 318.469}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 318.289}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 327.253}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 248.311}]}
{"cur_time": "2024-08-20 10:39:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 404.062}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 251.145}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 305.035}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 198.036}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 241.569}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 265.065}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 338.012}]}
{"cur_time": "2024-08-20 10:40:30", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 293.774}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 389.609}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 270.443}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 266.721}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 330.009}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 314.597}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 406.491}]}
{"cur_time": "2024-08-20 10:41:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 279.387}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 364.034}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 348.879}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 347.389}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 303.569}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 309.333}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 379.029}]}
{"cur_time": "2024-08-20 10:42:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 286.755}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 386.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 229.865}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 398.285}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 284.913}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 353.149}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 322.195}]}
{"cur_time": "2024-08-20 10:43:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 312.247}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 340.548}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 336.327}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 288.09}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 292.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 242.258}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 297.205}]}
{"cur_time": "2024-08-20 10:44:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 393.384}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 294.913}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 324.932}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 255.409}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 277.192}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 265.829}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 233.787}]}
{"cur_time": "2024-08-20 10:45:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 294.779}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 229.372}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 335.563}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 300.199}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 291.22}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 357.567}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 292.89}]}
{"cur_time": "2024-08-20 10:46:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 276.849}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 302.007}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 235.44}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 339.38}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 237.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 330.096}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 325.252}]}
{"cur_time": "2024-08-20 10:47:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 306.332}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 306.338}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 418.852}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 359.5}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 279.659}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 358.914}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 418.32}]}
{"cur_time": "2024-08-20 10:48:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 363.631}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 289.072}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 286.842}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 224.097}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 388.062}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 259.784}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 227.593}]}
{"cur_time": "2024-08-20 10:49:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 308.476}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 305.512}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 305.372}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 269.566}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 258.196}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 275.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 407.182}]}
{"cur_time": "2024-08-20 10:50:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 323.561}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 259.733}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 364.487}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 316.555}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 241.083}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 399.725}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 348.289}]}
{"cur_time": "2024-08-20 10:51:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 256.531}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 347.462}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 379.742}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 242.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 325.948}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 324.79}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 209.598}]}
{"cur_time": "2024-08-20 10:52:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 264.117}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 277.044}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 275.442}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 280.59}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 375.017}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 248.377}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 368.433}]}
{"cur_time": "2024-08-20 10:53:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 345.63}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 374.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 247.392}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 251.464}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 262.601}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 258.886}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 338.733}]}
{"cur_time": "2024-08-20 10:54:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 351.337}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 315.375}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 280.194}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 278.78}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 333.464}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 302.58}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 242.898}]}
{"cur_time": "2024-08-20 10:55:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 317.186}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 261.735}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 339.549}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 294.135}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 334.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 370.217}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 326.337}]}
{"cur_time": "2024-08-20 10:56:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 237.484}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 433.567}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 324.272}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 283.451}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 311.493}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 351.565}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 226.221}]}
{"cur_time": "2024-08-20 10:57:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 268.964}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 313.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 307.484}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 363.362}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 288.919}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 363.786}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 324.117}]}
{"cur_time": "2024-08-20 10:58:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 316.579}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 359.142}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 360.577}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 308.033}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 365.38}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 294.873}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 368.611}]}
{"cur_time": "2024-08-20 10:59:31", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 294.742}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 318.389}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 326.991}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 369.31}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 270.015}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 272.429}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 316.398}]}
{"cur_time": "2024-08-20 11:00:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 376.633}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 345.144}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 292.741}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 304.086}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 365.614}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 270.832}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 309.256}]}
{"cur_time": "2024-08-20 11:01:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 257.989}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 307.627}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 278.394}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 375.955}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 293.957}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 259.684}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 275.988}]}
{"cur_time": "2024-08-20 11:02:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 305.731}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 292.385}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 385.741}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 281.146}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 385.74}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 286.93}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 331.064}]}
{"cur_time": "2024-08-20 11:03:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 346.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 317.233}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 304.923}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 318.801}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 361.103}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 348.369}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 220.188}]}
{"cur_time": "2024-08-20 11:04:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99212646484375, "powerusage_W": 322.86}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.58392333984375, "powerusage_W": 349.991}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33392333984375, "powerusage_W": 358.126}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 295.333}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 392.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.06829833984375, "powerusage_W": 417.92}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.10931396484375, "powerusage_W": 244.19}]}
{"cur_time": "2024-08-20 11:05:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.469}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 89.964}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.976}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 87.345}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.858}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.99}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.695}]}
{"cur_time": "2024-08-20 11:06:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.31634521484375, "powerusage_W": 139.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.733}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 139.462}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 130.307}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 128.907}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 135.361}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.12884521484375, "powerusage_W": 130.831}]}
{"cur_time": "2024-08-20 11:07:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.59759521484375, "powerusage_W": 138.916}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.29876708984375, "powerusage_W": 134.058}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.98431396484375, "powerusage_W": 139.068}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.23626708984375, "powerusage_W": 129.83}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.23626708984375, "powerusage_W": 128.365}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.23626708984375, "powerusage_W": 134.865}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.00189208984375, "powerusage_W": 130.123}]}
{"cur_time": "2024-08-20 11:08:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.23431396484375, "powerusage_W": 288.308}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.04876708984375, "powerusage_W": 300.163}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99798583984375, "powerusage_W": 433.745}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 418.401}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.31243896484375, "powerusage_W": 299.809}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.30072021484375, "powerusage_W": 368.903}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.07806396484375, "powerusage_W": 316.312}]}
{"cur_time": "2024-08-20 11:09:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.23431396484375, "powerusage_W": 310.075}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08197021484375, "powerusage_W": 305.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.99993896484375, "powerusage_W": 412.947}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 286.437}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 327.774}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 277.154}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 231.526}]}
{"cur_time": "2024-08-20 11:10:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27142333984375, "powerusage_W": 272.84}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08197021484375, "powerusage_W": 331.63}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 334.966}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 255.827}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 385.411}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 325.369}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 383.512}]}
{"cur_time": "2024-08-20 11:11:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27142333984375, "powerusage_W": 360.345}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 305.898}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 406.425}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 285.896}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 309.944}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 355.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 230.735}]}
{"cur_time": "2024-08-20 11:12:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27142333984375, "powerusage_W": 301.924}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 256.558}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 291.42}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 251.971}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 350.154}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 337.296}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 306.422}]}
{"cur_time": "2024-08-20 11:13:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27142333984375, "powerusage_W": 325.815}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 364.679}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 292.714}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 319.812}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 233.733}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 255.941}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 245.086}]}
{"cur_time": "2024-08-20 11:14:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27337646484375, "powerusage_W": 340.836}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 255.959}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 252.436}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 243.948}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 280.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 311.364}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 244.489}]}
{"cur_time": "2024-08-20 11:15:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27337646484375, "powerusage_W": 439.941}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 340.482}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 324.771}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 303.93}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 295.59}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 327.259}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11517333984375, "powerusage_W": 336.129}]}
{"cur_time": "2024-08-20 11:16:32", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 288.196}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 339.065}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 305.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 383.206}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 253.314}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 334.646}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 215.754}]}
{"cur_time": "2024-08-20 11:17:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 284.557}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 372.71}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 329.433}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 352.017}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 360.429}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 310.394}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 334.982}]}
{"cur_time": "2024-08-20 11:18:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 276.605}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 303.489}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 266.944}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 298.406}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 344.728}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 344.273}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 309.304}]}
{"cur_time": "2024-08-20 11:19:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 406.093}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 330.867}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03509521484375, "powerusage_W": 270.198}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 259.021}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 225.681}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 257.708}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 334.688}]}
{"cur_time": "2024-08-20 11:20:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 334.195}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 401.745}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 341.117}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 299.141}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 226.879}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 286.645}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 324.324}]}
{"cur_time": "2024-08-20 11:21:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 439.85}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 276.57}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 335.866}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 271.962}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 268.555}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 280.873}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 369.983}]}
{"cur_time": "2024-08-20 11:22:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 255.517}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 390.287}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 324.516}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 238.937}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 407.744}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 250.203}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 319.609}]}
{"cur_time": "2024-08-20 11:23:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 356.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 284.167}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 294.975}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 280.301}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 315.434}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 253.556}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 374.935}]}
{"cur_time": "2024-08-20 11:24:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 214.898}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 347.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 350.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 309.962}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 230.443}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 399.434}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 359.193}]}
{"cur_time": "2024-08-20 11:25:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 375.157}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 253.576}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 315.955}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 222.005}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 226.195}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 201.817}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 359.356}]}
{"cur_time": "2024-08-20 11:26:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 322.803}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 322.19}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 257.8}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 319.786}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 287.165}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 371.99}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 347.818}]}
{"cur_time": "2024-08-20 11:27:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 395.171}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 260.765}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 301.939}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 300.997}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 229.799}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 280.598}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 395.159}]}
{"cur_time": "2024-08-20 11:28:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 297.341}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 342.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 299.085}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 275.34}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 346.165}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 286.404}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 261.692}]}
{"cur_time": "2024-08-20 11:29:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 360.488}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 286.242}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 263.598}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 260.002}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 415.821}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 276.673}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 212.67}]}
{"cur_time": "2024-08-20 11:30:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 294.888}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 384.156}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 304.935}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 284.112}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 304.542}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 287.677}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 307.34}]}
{"cur_time": "2024-08-20 11:31:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 261.832}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 347.696}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 406.26}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 338.747}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 297.529}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 343.29}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 307.797}]}
{"cur_time": "2024-08-20 11:32:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 336.375}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 259.74}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 379.371}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 287.69}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 319.694}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 255.377}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 366.104}]}
{"cur_time": "2024-08-20 11:33:33", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 264.217}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 320.638}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 337.931}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 290.08}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 286.054}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 239.074}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 312.999}]}
{"cur_time": "2024-08-20 11:34:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 339.792}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 331.915}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 380.058}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 356.867}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 272.142}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 262.501}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 304.572}]}
{"cur_time": "2024-08-20 11:35:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 301.704}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 274.762}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 343.905}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 296.089}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 297.861}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 339.402}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 345.373}]}
{"cur_time": "2024-08-20 11:36:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 343.529}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 398.894}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 244.646}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 277.847}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 238.686}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 253.368}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 312.785}]}
{"cur_time": "2024-08-20 11:37:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 216.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 339.117}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 315.254}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 328.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 334.681}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 270.751}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 255.987}]}
{"cur_time": "2024-08-20 11:38:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 317.468}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 332.213}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 365.603}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 280.23}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 309.503}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 332.658}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 268.371}]}
{"cur_time": "2024-08-20 11:39:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 307.423}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 303.202}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03704833984375, "powerusage_W": 316.972}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 310.981}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 312.77}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 284.481}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 290.778}]}
{"cur_time": "2024-08-20 11:40:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 246.72}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 316.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 358.448}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 253.14}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 258.799}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 403.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 321.982}]}
{"cur_time": "2024-08-20 11:41:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 271.16}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 319.742}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 422.031}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 255.206}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 281.459}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 284.34}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 367.172}]}
{"cur_time": "2024-08-20 11:42:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 246.444}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 295.692}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 223.011}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 294.421}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 368.646}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 264.342}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 188.452}]}
{"cur_time": "2024-08-20 11:43:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 286.595}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 289.851}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 241.203}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 250.098}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 331.096}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 256.76}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 385.892}]}
{"cur_time": "2024-08-20 11:44:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 337.338}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 310.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 305.853}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 341.333}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 315.892}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 240.168}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 387.571}]}
{"cur_time": "2024-08-20 11:45:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 245.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 280.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 319.823}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 298.958}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 424.88}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 242.686}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 298.468}]}
{"cur_time": "2024-08-20 11:46:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.27532958984375, "powerusage_W": 382.237}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.08782958984375, "powerusage_W": 323.429}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.03900146484375, "powerusage_W": 302.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33197021484375, "powerusage_W": 352.319}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.35150146484375, "powerusage_W": 311.725}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.33978271484375, "powerusage_W": 347.404}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.11712646484375, "powerusage_W": 335.462}]}
{"cur_time": "2024-08-20 11:47:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.742}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 89.228}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.993}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.306}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.168}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.771}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.643}]}
{"cur_time": "2024-08-20 11:48:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.08001708984375, "powerusage_W": 412.968}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.796}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.773}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.022}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.852}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.729}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.46}]}
{"cur_time": "2024-08-20 11:49:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 430.617}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.719}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.677}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.894}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.779}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.609}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.28}]}
{"cur_time": "2024-08-20 11:50:34", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 286.054}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.68}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.648}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.961}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.746}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.529}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.269}]}
{"cur_time": "2024-08-20 11:51:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 462.456}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.665}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.532}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.882}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.543}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.525}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.145}]}
{"cur_time": "2024-08-20 11:52:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 430.638}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.619}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.607}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.949}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.492}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.544}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.201}]}
{"cur_time": "2024-08-20 11:53:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 405.351}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.644}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.643}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.898}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.477}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.512}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.159}]}
{"cur_time": "2024-08-20 11:54:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 403.467}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.61}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.557}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.87}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.525}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.477}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.157}]}
{"cur_time": "2024-08-20 11:55:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 402.792}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.562}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.574}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.841}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.491}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.475}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.132}]}
{"cur_time": "2024-08-20 11:56:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 400.012}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.556}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.571}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.888}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.518}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.46}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.132}]}
{"cur_time": "2024-08-20 11:57:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 390.696}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.491}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.553}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.816}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.487}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.487}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.072}]}
{"cur_time": "2024-08-20 11:58:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 362.52}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.581}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.583}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.862}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.43}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.491}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.103}]}
{"cur_time": "2024-08-20 11:59:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 344.57}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.489}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.61}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.85}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.434}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.458}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.104}]}
{"cur_time": "2024-08-20 12:00:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 419.455}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.4}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.423}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.688}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.407}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.455}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.041}]}
{"cur_time": "2024-08-20 12:01:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 418.401}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.505}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.512}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.881}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.596}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.434}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.104}]}
{"cur_time": "2024-08-20 12:02:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 384.107}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.477}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.499}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.781}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.583}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.425}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.095}]}
{"cur_time": "2024-08-20 12:03:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 433.679}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.41}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.481}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.736}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.488}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.398}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.06}]}
{"cur_time": "2024-08-20 12:04:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 395.48}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.409}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.556}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.713}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.526}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.426}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.073}]}
{"cur_time": "2024-08-20 12:05:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 350.869}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.462}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.527}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.741}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.596}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.426}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.096}]}
{"cur_time": "2024-08-20 12:06:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 331.148}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.444}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.568}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.785}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.389}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.391}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}]}
{"cur_time": "2024-08-20 12:07:35", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 377.099}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.481}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.541}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.822}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.437}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.157}]}
{"cur_time": "2024-08-20 12:08:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 388.399}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.423}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.554}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.753}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.348}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.483}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.094}]}
{"cur_time": "2024-08-20 12:09:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 403.966}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.451}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.541}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.74}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.393}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.371}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.108}]}
{"cur_time": "2024-08-20 12:10:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 443.126}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.419}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.472}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.713}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.424}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.081}]}
{"cur_time": "2024-08-20 12:11:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 336.605}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.441}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.478}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.692}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.413}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 12:12:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 378.649}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.332}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.439}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.766}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.499}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.348}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.044}]}
{"cur_time": "2024-08-20 12:13:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 431.531}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.446}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.443}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.781}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.556}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.336}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.095}]}
{"cur_time": "2024-08-20 12:14:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.458}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.524}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.513}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.833}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.577}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.46}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 12:15:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.59368896484375, "powerusage_W": 441.08}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.385}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.46}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.64}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.526}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.398}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.057}]}
{"cur_time": "2024-08-20 12:16:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 405.05}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.43}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.536}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.795}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.519}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.359}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.08}]}
{"cur_time": "2024-08-20 12:17:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 391.341}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.488}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.565}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.765}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.558}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.331}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.094}]}
{"cur_time": "2024-08-20 12:18:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 434.286}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.396}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.493}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.695}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.348}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.4}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.071}]}
{"cur_time": "2024-08-20 12:19:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 362.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.371}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.408}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.772}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.298}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.359}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 12:20:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 365.559}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.464}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.432}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.741}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.346}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 12:21:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 340.73}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.369}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.45}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.732}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.311}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.323}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.007}]}
{"cur_time": "2024-08-20 12:22:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 386.372}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.374}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.414}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.733}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.311}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.349}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.022}]}
{"cur_time": "2024-08-20 12:23:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 369.671}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.375}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.502}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.76}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.356}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.424}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.114}]}
{"cur_time": "2024-08-20 12:24:36", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 397.895}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.351}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.494}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.727}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.367}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.365}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.099}]}
{"cur_time": "2024-08-20 12:25:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 382.177}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.405}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.436}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.78}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.391}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.105}]}
{"cur_time": "2024-08-20 12:26:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 357.816}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.377}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.443}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.691}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.355}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.011}]}
{"cur_time": "2024-08-20 12:27:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 391.726}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.346}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.362}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.639}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.428}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.294}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.969}]}
{"cur_time": "2024-08-20 12:28:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 349.201}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.357}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.497}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.676}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.454}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.353}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.067}]}
{"cur_time": "2024-08-20 12:29:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 407.475}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.397}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.469}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.512}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.404}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.042}]}
{"cur_time": "2024-08-20 12:30:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 434.5}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.387}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.509}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.728}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.533}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.352}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.994}]}
{"cur_time": "2024-08-20 12:31:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 354.595}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.39}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.421}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.67}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.506}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.339}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.06}]}
{"cur_time": "2024-08-20 12:32:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 383.671}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.356}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.429}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.732}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.539}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.402}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.089}]}
{"cur_time": "2024-08-20 12:33:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 385.846}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.418}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.529}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.745}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.393}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.47}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.089}]}
{"cur_time": "2024-08-20 12:34:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 391.899}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.404}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.465}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.718}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.322}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.99}]}
{"cur_time": "2024-08-20 12:35:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 407.426}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.37}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.38}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.709}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.379}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.027}]}
{"cur_time": "2024-08-20 12:36:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 314.981}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.386}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.475}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.745}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.374}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.414}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.043}]}
{"cur_time": "2024-08-20 12:37:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 346.253}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.416}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.444}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.749}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.514}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.343}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 12:38:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 376.291}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.518}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.539}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.716}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.545}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.431}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.124}]}
{"cur_time": "2024-08-20 12:39:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 378.551}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.456}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.397}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.722}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.571}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.485}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}]}
{"cur_time": "2024-08-20 12:40:37", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 380.073}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.377}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.456}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.707}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.577}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.422}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.057}]}
{"cur_time": "2024-08-20 12:41:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 436.817}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.456}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.457}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.703}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.469}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.353}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.036}]}
{"cur_time": "2024-08-20 12:42:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.75775146484375, "powerusage_W": 138.903}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.405}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.255}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.799}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.441}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.373}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 12:43:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.08001708984375, "powerusage_W": 408.401}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.365}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.426}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.667}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.329}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.385}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.981}]}
{"cur_time": "2024-08-20 12:44:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 430.623}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.439}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.523}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.872}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.413}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.452}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.153}]}
{"cur_time": "2024-08-20 12:45:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 392.413}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.496}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.564}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.822}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.428}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.405}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.095}]}
{"cur_time": "2024-08-20 12:46:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 396.601}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.452}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.552}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.813}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.409}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.409}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 12:47:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 368.408}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.47}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.539}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.701}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.374}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.401}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.102}]}
{"cur_time": "2024-08-20 12:48:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 345.17}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.437}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.281}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.79}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.361}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.394}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 12:49:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 351.581}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.373}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.438}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.766}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.383}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.331}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.007}]}
{"cur_time": "2024-08-20 12:50:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 359.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.451}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.559}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.794}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.368}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.382}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 12:51:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 407.966}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.414}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.505}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.741}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.374}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.35}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 12:52:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 392.877}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.377}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.319}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.743}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.558}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.422}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.101}]}
{"cur_time": "2024-08-20 12:53:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 414.997}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.37}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.511}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.759}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.583}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.424}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.056}]}
{"cur_time": "2024-08-20 12:54:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 335.653}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.403}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.433}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.712}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.495}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.359}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.015}]}
{"cur_time": "2024-08-20 12:55:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 436.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.414}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.454}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.723}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.519}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.362}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.088}]}
{"cur_time": "2024-08-20 12:56:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 374.615}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.439}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.562}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.788}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.558}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.399}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.102}]}
{"cur_time": "2024-08-20 12:57:38", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 389.829}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.53}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.542}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.735}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.552}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.433}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.066}]}
{"cur_time": "2024-08-20 12:58:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 430.374}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.458}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.492}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.72}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.564}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.467}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.149}]}
{"cur_time": "2024-08-20 12:59:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 389.853}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.337}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.498}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.749}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.577}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.401}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.095}]}
{"cur_time": "2024-08-20 13:00:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 347.647}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.525}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.519}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.384}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}]}
{"cur_time": "2024-08-20 13:01:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 381.245}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.457}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.548}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.789}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.59}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.433}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.093}]}
{"cur_time": "2024-08-20 13:02:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 351.011}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.365}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.429}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.765}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.35}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.443}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.073}]}
{"cur_time": "2024-08-20 13:03:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 405.497}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.407}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.484}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.719}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.336}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.318}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.037}]}
{"cur_time": "2024-08-20 13:04:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 375.481}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.357}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.475}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.761}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.304}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.327}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.992}]}
{"cur_time": "2024-08-20 13:05:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 436.879}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.345}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.423}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.684}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.266}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.243}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.977}]}
{"cur_time": "2024-08-20 13:06:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 337.729}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.357}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.413}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.637}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.434}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.347}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.974}]}
{"cur_time": "2024-08-20 13:07:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 393.266}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.397}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.438}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.69}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.434}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.324}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.908}]}
{"cur_time": "2024-08-20 13:08:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.23822021484375, "powerusage_W": 429.493}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.372}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.399}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.616}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.402}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.208}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.951}]}
{"cur_time": "2024-08-20 13:09:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.437}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 134.778}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 138.806}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 129.194}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 127.485}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 134.703}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.10345458984375, "powerusage_W": 130.083}]}
{"cur_time": "2024-08-20 13:10:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.31634521484375, "powerusage_W": 139.693}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.44}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 138.977}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 129.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 127.815}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.36322021484375, "powerusage_W": 134.874}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.12884521484375, "powerusage_W": 130.282}]}
{"cur_time": "2024-08-20 13:11:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.90618896484375, "powerusage_W": 139.387}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.46478271484375, "powerusage_W": 134.528}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.46673583984375, "powerusage_W": 139.016}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.46673583984375, "powerusage_W": 129.367}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.63861083984375, "powerusage_W": 127.856}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.46673583984375, "powerusage_W": 134.85}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.23236083984375, "powerusage_W": 130.301}]}
{"cur_time": "2024-08-20 13:12:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.16986083984375, "powerusage_W": 369.471}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59759521484375, "powerusage_W": 366.614}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59759521484375, "powerusage_W": 411.082}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.67376708984375, "powerusage_W": 449.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 389.151}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.21868896484375, "powerusage_W": 458.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98236083984375, "powerusage_W": 399.678}]}
{"cur_time": "2024-08-20 13:13:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.16986083984375, "powerusage_W": 475.016}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59759521484375, "powerusage_W": 399.126}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59954833984375, "powerusage_W": 409.524}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.67376708984375, "powerusage_W": 286.393}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 347.578}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.21868896484375, "powerusage_W": 317.458}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98431396484375, "powerusage_W": 400.938}]}
{"cur_time": "2024-08-20 13:14:39", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.16986083984375, "powerusage_W": 301.147}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59954833984375, "powerusage_W": 242.842}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 191.496}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 492.886}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 218.334}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.21868896484375, "powerusage_W": 231.844}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98431396484375, "powerusage_W": 210.889}]}
{"cur_time": "2024-08-20 13:15:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.17181396484375, "powerusage_W": 445.876}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59954833984375, "powerusage_W": 318.732}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 375.397}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 367.854}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 306.409}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 384.606}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98431396484375, "powerusage_W": 402.234}]}
{"cur_time": "2024-08-20 13:16:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.17181396484375, "powerusage_W": 435.289}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.59954833984375, "powerusage_W": 418.333}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 384.349}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 390.956}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 376.519}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 337.333}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98431396484375, "powerusage_W": 243.11}]}
{"cur_time": "2024-08-20 13:17:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66009521484375, "powerusage_W": 355.783}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 372.261}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 373.744}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 287.396}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 322.401}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 371.455}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98431396484375, "powerusage_W": 262.446}]}
{"cur_time": "2024-08-20 13:18:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66009521484375, "powerusage_W": 371.003}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 341.582}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 377.325}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 377.72}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 452.237}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44525146484375, "powerusage_W": 347.247}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98626708984375, "powerusage_W": 355.46}]}
{"cur_time": "2024-08-20 13:19:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 491.719}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 259.722}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 323.26}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 227.334}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 325.84}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44525146484375, "powerusage_W": 316.298}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98626708984375, "powerusage_W": 302.183}]}
{"cur_time": "2024-08-20 13:20:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 393.22}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 463.124}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 433.355}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 360.484}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 375.002}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44525146484375, "powerusage_W": 359.451}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98626708984375, "powerusage_W": 480.105}]}
{"cur_time": "2024-08-20 13:21:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 486.51}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 429.12}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 297.202}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 308.017}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 353.425}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44525146484375, "powerusage_W": 368.502}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.98626708984375, "powerusage_W": 360.381}]}
{"cur_time": "2024-08-20 13:22:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 367.978}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 425.184}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 355.871}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 329.042}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 376.426}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44525146484375, "powerusage_W": 440.602}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21087646484375, "powerusage_W": 352.415}]}
{"cur_time": "2024-08-20 13:23:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 384.431}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 328.483}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 307.361}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 379.639}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 362.252}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.90032958984375, "powerusage_W": 459.22}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21087646484375, "powerusage_W": 365.137}]}
{"cur_time": "2024-08-20 13:24:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 324.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 380.133}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 379.327}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 324.861}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 374.339}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.90032958984375, "powerusage_W": 403.194}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21087646484375, "powerusage_W": 430.946}]}
{"cur_time": "2024-08-20 13:25:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.66204833984375, "powerusage_W": 449.916}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08782958984375, "powerusage_W": 352.753}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.08978271484375, "powerusage_W": 346.117}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.16204833984375, "powerusage_W": 319.174}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 26.22064208984375, "powerusage_W": 442.782}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.90032958984375, "powerusage_W": 335.987}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.21087646484375, "powerusage_W": 343.879}]}
{"cur_time": "2024-08-20 13:26:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.03857421875, "powerusage_W": 328.966}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.59521484375, "powerusage_W": 410.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.59716796875, "powerusage_W": 352.456}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.66943359375, "powerusage_W": 328.803}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.72802734375, "powerusage_W": 452.933}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.66748046875, "powerusage_W": 389.479}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.72412109375, "powerusage_W": 361.34}]}
{"cur_time": "2024-08-20 13:27:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.03857421875, "powerusage_W": 429.666}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.59521484375, "powerusage_W": 403.378}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.59716796875, "powerusage_W": 427.924}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.66943359375, "powerusage_W": 438.677}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.72802734375, "powerusage_W": 396.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.66748046875, "powerusage_W": 455.352}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.72412109375, "powerusage_W": 307.118}]}
{"cur_time": "2024-08-20 13:28:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.41510009765625, "powerusage_W": 362.241}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10260009765625, "powerusage_W": 352.818}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10455322265625, "powerusage_W": 334.218}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.17681884765625, "powerusage_W": 480.348}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.23541259765625, "powerusage_W": 477.659}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.94635009765625, "powerusage_W": 351.217}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.23736572265625, "powerusage_W": 351.144}]}
{"cur_time": "2024-08-20 13:29:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.41510009765625, "powerusage_W": 350.54}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10260009765625, "powerusage_W": 396.178}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10455322265625, "powerusage_W": 325.316}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.17681884765625, "powerusage_W": 395.943}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.23541259765625, "powerusage_W": 461.886}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.94635009765625, "powerusage_W": 400.21}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.23736572265625, "powerusage_W": 398.733}]}
{"cur_time": "2024-08-20 13:30:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.41510009765625, "powerusage_W": 350.604}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10260009765625, "powerusage_W": 352.428}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.10455322265625, "powerusage_W": 493.106}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.17681884765625, "powerusage_W": 308.007}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.23541259765625, "powerusage_W": 356.278}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.94635009765625, "powerusage_W": 376.207}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.23736572265625, "powerusage_W": 400.133}]}
{"cur_time": "2024-08-20 13:31:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 389.544}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 322.327}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 292.826}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 393.918}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1978759765625, "powerusage_W": 400.961}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 404.964}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 359.562}]}
{"cur_time": "2024-08-20 13:32:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 356.752}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 374.037}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 287.049}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 419.302}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1978759765625, "powerusage_W": 320.8}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 291.654}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 447.368}]}
{"cur_time": "2024-08-20 13:33:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 405.664}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 291.636}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 353.443}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 343.116}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1978759765625, "powerusage_W": 342.533}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 316.653}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 502.04}]}
{"cur_time": "2024-08-20 13:34:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 307.267}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 469.755}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 383.242}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 361.289}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1978759765625, "powerusage_W": 275.067}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 307.563}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 332.605}]}
{"cur_time": "2024-08-20 13:35:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 321.295}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 311.985}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 331.231}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 496.917}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1978759765625, "powerusage_W": 470.188}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 317.338}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 330.703}]}
{"cur_time": "2024-08-20 13:36:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.7935791015625, "powerusage_W": 278.602}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 308.693}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.6119384765625, "powerusage_W": 301.403}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6842041015625, "powerusage_W": 332.123}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.6861572265625, "powerusage_W": 281.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.2271728515625, "powerusage_W": 421.78}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7506103515625, "powerusage_W": 422.921}]}
{"cur_time": "2024-08-20 13:37:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.59283447265625, "powerusage_W": 309.872}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 368.232}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 261.953}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89556884765625, "powerusage_W": 318.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 409.778}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.58892822265625, "powerusage_W": 298.271}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.23736572265625, "powerusage_W": 364.513}]}
{"cur_time": "2024-08-20 13:38:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.32330322265625, "powerusage_W": 486.159}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 446.002}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 428.476}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89556884765625, "powerusage_W": 379.307}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 388.668}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.58892822265625, "powerusage_W": 394.251}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.69244384765625, "powerusage_W": 369.309}]}
{"cur_time": "2024-08-20 13:39:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 45.06549072265625, "powerusage_W": 223.682}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 253.55}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.82330322265625, "powerusage_W": 246.218}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89556884765625, "powerusage_W": 236.053}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 205.721}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.58892822265625, "powerusage_W": 205.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 466.393}]}
{"cur_time": "2024-08-20 13:40:40", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.8853759765625, "powerusage_W": 302.925}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 410.534}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 280.695}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 218.392}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 200.247}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 358.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 187.307}]}
{"cur_time": "2024-08-20 13:41:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.4654541015625, "powerusage_W": 432.349}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 398.44}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 264.521}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 385.093}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 320.937}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 333.252}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 319.152}]}
{"cur_time": "2024-08-20 13:42:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.4654541015625, "powerusage_W": 433.465}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 381.475}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 351.156}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 354.564}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 279.97}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 301.857}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 329.119}]}
{"cur_time": "2024-08-20 13:43:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.4654541015625, "powerusage_W": 449.039}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 224.594}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 251.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 205.247}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 241.787}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 269.461}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 307.89}]}
{"cur_time": "2024-08-20 13:44:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 443.199}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 317.155}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 395.734}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 367.223}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 291.856}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 376.144}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 360.741}]}
{"cur_time": "2024-08-20 13:45:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 386.89}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 421.425}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 369.853}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 357.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 311.812}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.62799072265625, "powerusage_W": 360.683}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 344.439}]}
{"cur_time": "2024-08-20 13:46:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 426.736}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 365.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 408.319}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 378.913}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 309.188}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 425.892}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 355.0}]}
{"cur_time": "2024-08-20 13:47:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 372.497}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 303.661}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 373.706}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 412.666}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 277.462}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 330.214}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 281.648}]}
{"cur_time": "2024-08-20 13:48:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 425.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 365.302}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 419.261}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 339.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 412.607}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 406.943}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 358.195}]}
{"cur_time": "2024-08-20 13:49:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 440.136}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 389.013}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 389.025}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 356.881}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 299.072}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 421.86}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 439.737}]}
{"cur_time": "2024-08-20 13:50:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 375.318}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 388.176}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 270.567}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 315.579}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 283.335}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 469.787}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 353.958}]}
{"cur_time": "2024-08-20 13:51:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.5787353515625, "powerusage_W": 449.209}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 267.138}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 224.072}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 188.196}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 292.21}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 256.133}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 254.642}]}
{"cur_time": "2024-08-20 13:52:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 397.918}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 344.309}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 409.891}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 363.282}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 335.055}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 379.721}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 279.402}]}
{"cur_time": "2024-08-20 13:53:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 388.767}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 422.12}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 354.002}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 366.917}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 325.727}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 359.711}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 438.775}]}
{"cur_time": "2024-08-20 13:54:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 381.756}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 375.381}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 319.466}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 322.654}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 284.747}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 397.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 250.496}]}
{"cur_time": "2024-08-20 13:55:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 421.926}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 338.184}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 311.029}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 324.536}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 337.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 269.925}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 253.981}]}
{"cur_time": "2024-08-20 13:56:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 415.769}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 309.311}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 403.054}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 330.164}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 372.985}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 351.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 343.279}]}
{"cur_time": "2024-08-20 13:57:41", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 421.827}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 322.182}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 352.411}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 351.86}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 318.934}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 322.62}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 347.894}]}
{"cur_time": "2024-08-20 13:58:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.3248291015625, "powerusage_W": 398.225}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 355.296}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 260.14}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 307.15}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 303.483}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 253.787}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 235.464}]}
{"cur_time": "2024-08-20 13:59:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 429.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 352.65}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 335.658}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 233.54}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 238.811}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 416.16}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 393.007}]}
{"cur_time": "2024-08-20 14:00:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 408.385}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 273.18}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 369.129}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 261.212}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 349.477}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 382.437}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 359.978}]}
{"cur_time": "2024-08-20 14:01:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 431.389}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 346.039}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 422.149}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 415.171}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 383.695}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 367.707}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 289.973}]}
{"cur_time": "2024-08-20 14:02:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 399.412}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 360.662}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 414.983}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 466.779}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 389.156}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 335.332}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 375.311}]}
{"cur_time": "2024-08-20 14:03:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 304.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 285.245}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 455.226}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 318.032}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 278.065}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 236.963}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 373.723}]}
{"cur_time": "2024-08-20 14:04:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 374.08}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 300.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 330.258}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 320.163}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 491.652}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 316.77}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 265.174}]}
{"cur_time": "2024-08-20 14:05:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 379.151}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 373.864}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 402.875}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 464.757}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 260.461}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 301.291}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 322.211}]}
{"cur_time": "2024-08-20 14:06:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.6431884765625, "powerusage_W": 394.879}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 311.281}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 290.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 307.512}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 344.75}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 391.81}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 290.541}]}
{"cur_time": "2024-08-20 14:07:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 370.806}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 411.841}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94049072265625, "powerusage_W": 406.517}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01275634765625, "powerusage_W": 304.301}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.01470947265625, "powerusage_W": 322.466}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.66705322265625, "powerusage_W": 400.853}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 282.299}]}
{"cur_time": "2024-08-20 14:08:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 370.215}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13775634765625, "powerusage_W": 333.401}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13580322265625, "powerusage_W": 279.743}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.20806884765625, "powerusage_W": 329.904}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.21002197265625, "powerusage_W": 374.675}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78424072265625, "powerusage_W": 409.762}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 452.427}]}
{"cur_time": "2024-08-20 14:09:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 388.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13775634765625, "powerusage_W": 306.334}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13580322265625, "powerusage_W": 407.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.20806884765625, "powerusage_W": 296.559}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.21002197265625, "powerusage_W": 332.51}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78424072265625, "powerusage_W": 399.754}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 438.672}]}
{"cur_time": "2024-08-20 14:10:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 443.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13775634765625, "powerusage_W": 282.945}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13580322265625, "powerusage_W": 235.635}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.20806884765625, "powerusage_W": 359.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.21002197265625, "powerusage_W": 273.1}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78424072265625, "powerusage_W": 262.09}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 245.643}]}
{"cur_time": "2024-08-20 14:11:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 441.173}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13775634765625, "powerusage_W": 253.403}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13580322265625, "powerusage_W": 288.353}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.20806884765625, "powerusage_W": 256.517}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.21002197265625, "powerusage_W": 308.637}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78424072265625, "powerusage_W": 259.465}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 286.37}]}
{"cur_time": "2024-08-20 14:12:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.6236572265625, "powerusage_W": 396.377}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13775634765625, "powerusage_W": 378.256}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.13580322265625, "powerusage_W": 295.505}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.20806884765625, "powerusage_W": 315.236}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.21002197265625, "powerusage_W": 316.08}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78424072265625, "powerusage_W": 363.496}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.18072509765625, "powerusage_W": 307.574}]}
{"cur_time": "2024-08-20 14:13:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.54986572265625, "powerusage_W": 339.372}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.15142822265625, "powerusage_W": 161.752}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.14947509765625, "powerusage_W": 157.424}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.14947509765625, "powerusage_W": 153.735}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.14947509765625, "powerusage_W": 152.41}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.49908447265625, "powerusage_W": 152.832}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 2.12994384765625, "powerusage_W": 131.003}]}
{"cur_time": "2024-08-20 14:14:42", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.6998291015625, "powerusage_W": 203.144}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 8.41119384765625, "powerusage_W": 156.173}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 8.40924072265625, "powerusage_W": 154.978}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 8.40924072265625, "powerusage_W": 146.84}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 8.40924072265625, "powerusage_W": 146.877}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 7.75885009765625, "powerusage_W": 150.768}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.15533447265625, "powerusage_W": 130.096}]}
{"cur_time": "2024-08-20 14:15:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3150634765625, "powerusage_W": 432.274}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40728759765625, "powerusage_W": 357.019}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02447509765625, "powerusage_W": 385.283}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.47955322265625, "powerusage_W": 354.488}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02642822265625, "powerusage_W": 381.401}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37408447265625, "powerusage_W": 297.241}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77056884765625, "powerusage_W": 301.472}]}
{"cur_time": "2024-08-20 14:16:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3150634765625, "powerusage_W": 430.196}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40728759765625, "powerusage_W": 356.196}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02447509765625, "powerusage_W": 361.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48150634765625, "powerusage_W": 270.519}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 269.507}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37603759765625, "powerusage_W": 288.707}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 360.499}]}
{"cur_time": "2024-08-20 14:17:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3150634765625, "powerusage_W": 380.568}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40924072265625, "powerusage_W": 458.26}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02642822265625, "powerusage_W": 343.206}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48150634765625, "powerusage_W": 449.758}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 343.306}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37603759765625, "powerusage_W": 301.166}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 434.756}]}
{"cur_time": "2024-08-20 14:18:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3170166015625, "powerusage_W": 447.356}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40924072265625, "powerusage_W": 230.633}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51470947265625, "powerusage_W": 266.655}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 222.916}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 197.672}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37603759765625, "powerusage_W": 214.768}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 191.835}]}
{"cur_time": "2024-08-20 14:19:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3170166015625, "powerusage_W": 434.758}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40924072265625, "powerusage_W": 332.797}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51470947265625, "powerusage_W": 341.916}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 279.146}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 333.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37799072265625, "powerusage_W": 265.872}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 365.759}]}
{"cur_time": "2024-08-20 14:20:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3170166015625, "powerusage_W": 395.376}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40924072265625, "powerusage_W": 415.617}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51470947265625, "powerusage_W": 382.887}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 311.814}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 415.196}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37799072265625, "powerusage_W": 397.823}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 395.552}]}
{"cur_time": "2024-08-20 14:21:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.3170166015625, "powerusage_W": 424.851}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.40924072265625, "powerusage_W": 332.395}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 339.383}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 340.025}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 285.21}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37799072265625, "powerusage_W": 300.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 273.494}]}
{"cur_time": "2024-08-20 14:22:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8052978515625, "powerusage_W": 427.603}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 425.941}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 356.97}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 302.532}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 349.841}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.37799072265625, "powerusage_W": 362.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77252197265625, "powerusage_W": 371.834}]}
{"cur_time": "2024-08-20 14:23:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8052978515625, "powerusage_W": 317.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 326.617}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 274.82}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 257.646}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 455.178}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 453.735}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77447509765625, "powerusage_W": 261.655}]}
{"cur_time": "2024-08-20 14:24:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 394.263}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 357.256}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 330.821}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 284.869}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 325.676}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 321.769}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77447509765625, "powerusage_W": 354.402}]}
{"cur_time": "2024-08-20 14:25:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 453.407}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 247.451}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 257.131}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 261.875}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 259.892}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 282.938}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77447509765625, "powerusage_W": 268.627}]}
{"cur_time": "2024-08-20 14:26:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 390.723}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 313.533}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 313.128}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 471.12}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 295.104}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 332.381}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.77447509765625, "powerusage_W": 378.285}]}
{"cur_time": "2024-08-20 14:27:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 356.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 416.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 370.909}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 293.731}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 424.633}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 345.107}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 411.191}]}
{"cur_time": "2024-08-20 14:28:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 342.961}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 439.647}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 264.202}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 412.038}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 253.989}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.60260009765625, "powerusage_W": 445.675}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 299.599}]}
{"cur_time": "2024-08-20 14:29:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 424.872}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 384.254}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 283.275}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 277.982}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 280.985}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.05767822265625, "powerusage_W": 351.244}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 241.854}]}
{"cur_time": "2024-08-20 14:30:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 408.167}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 357.054}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 344.242}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 273.871}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 306.034}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.05767822265625, "powerusage_W": 361.593}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 315.165}]}
{"cur_time": "2024-08-20 14:31:43", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 413.649}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 427.868}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 282.401}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 363.229}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 344.454}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.05767822265625, "powerusage_W": 345.848}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 423.766}]}
{"cur_time": "2024-08-20 14:32:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 451.072}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 367.475}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 404.365}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 276.8}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 394.027}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.05767822265625, "powerusage_W": 329.015}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 362.761}]}
{"cur_time": "2024-08-20 14:33:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 389.33}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 404.203}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 257.8}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 287.231}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 293.756}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 328.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 350.999}]}
{"cur_time": "2024-08-20 14:34:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 419.441}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 349.788}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 358.783}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 275.037}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 398.762}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 332.279}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 269.215}]}
{"cur_time": "2024-08-20 14:35:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 413.746}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 267.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 435.903}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 301.246}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 336.882}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 275.875}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 252.746}]}
{"cur_time": "2024-08-20 14:36:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 394.734}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 282.403}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 383.303}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 361.82}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 452.368}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 335.968}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 386.11}]}
{"cur_time": "2024-08-20 14:37:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 358.752}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 457.28}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 339.994}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 343.533}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 345.361}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 368.569}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 259.269}]}
{"cur_time": "2024-08-20 14:38:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 395.044}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 352.507}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 482.028}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 295.767}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.02838134765625, "powerusage_W": 375.563}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 334.745}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 378.387}]}
{"cur_time": "2024-08-20 14:39:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 437.793}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 402.459}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 459.15}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 437.65}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 431.217}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 286.698}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 280.079}]}
{"cur_time": "2024-08-20 14:40:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 388.861}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 411.524}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 350.118}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 291.313}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 394.861}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 280.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 400.193}]}
{"cur_time": "2024-08-20 14:41:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 437.064}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 382.971}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 345.15}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 282.892}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 304.816}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 375.777}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 344.323}]}
{"cur_time": "2024-08-20 14:42:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 411.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 445.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 443.423}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 381.526}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 390.238}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 402.861}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 359.869}]}
{"cur_time": "2024-08-20 14:43:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 329.832}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 367.654}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 411.35}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 377.301}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 362.258}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 306.796}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 253.175}]}
{"cur_time": "2024-08-20 14:44:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 402.462}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 450.313}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 301.355}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 350.653}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 276.602}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 295.309}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 295.365}]}
{"cur_time": "2024-08-20 14:45:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 428.481}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 350.364}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 358.844}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 288.06}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.48345947265625, "powerusage_W": 288.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 355.419}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 323.96}]}
{"cur_time": "2024-08-20 14:46:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 393.542}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 283.932}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 349.684}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 377.244}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 319.1}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 476.867}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.99908447265625, "powerusage_W": 255.076}]}
{"cur_time": "2024-08-20 14:47:44", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 392.349}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 302.077}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 274.581}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 278.069}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 283.147}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 364.003}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.45416259765625, "powerusage_W": 348.865}]}
{"cur_time": "2024-08-20 14:48:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 373.962}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 459.346}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 395.594}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 386.92}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 318.173}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 322.162}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.45416259765625, "powerusage_W": 346.675}]}
{"cur_time": "2024-08-20 14:49:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 348.851}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 378.208}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 239.261}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 487.987}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 236.526}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 231.299}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 253.637}]}
{"cur_time": "2024-08-20 14:50:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 441.264}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 239.532}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 389.285}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 279.656}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 300.612}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 246.591}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 273.709}]}
{"cur_time": "2024-08-20 14:51:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 430.31}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 315.571}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 415.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 358.627}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 345.912}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 398.727}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 409.63}]}
{"cur_time": "2024-08-20 14:52:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 359.595}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 396.146}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 398.773}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 382.15}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 416.651}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 404.031}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 334.292}]}
{"cur_time": "2024-08-20 14:53:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 390.783}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 313.43}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 396.156}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 386.97}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 374.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 408.252}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 365.509}]}
{"cur_time": "2024-08-20 14:54:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 359.787}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 437.41}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 249.989}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 334.412}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 381.423}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 333.429}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 358.426}]}
{"cur_time": "2024-08-20 14:55:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 411.413}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 437.646}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 342.427}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 312.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 249.818}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 306.953}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 345.201}]}
{"cur_time": "2024-08-20 14:56:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 427.555}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 389.635}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 330.46}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 406.862}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 415.787}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 329.719}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 325.651}]}
{"cur_time": "2024-08-20 14:57:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 404.987}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 339.909}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 363.347}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 280.289}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 274.624}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 335.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 274.454}]}
{"cur_time": "2024-08-20 14:58:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 396.283}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 267.132}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 275.766}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 282.794}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 393.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 443.544}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 352.493}]}
{"cur_time": "2024-08-20 14:59:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 452.277}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 294.245}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 375.346}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 230.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 262.577}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 321.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 214.857}]}
{"cur_time": "2024-08-20 15:00:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 450.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 354.823}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 352.893}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 335.22}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 224.071}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 343.374}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 307.96}]}
{"cur_time": "2024-08-20 15:01:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 368.551}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 339.993}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 445.502}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 316.957}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 400.753}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 344.437}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 447.027}]}
{"cur_time": "2024-08-20 15:02:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 433.054}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 462.031}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 273.154}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 440.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 298.045}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 344.158}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 321.519}]}
{"cur_time": "2024-08-20 15:03:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 363.514}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 459.065}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 307.46}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 258.871}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 203.472}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 282.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 320.304}]}
{"cur_time": "2024-08-20 15:04:45", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 341.085}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89752197265625, "powerusage_W": 272.702}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 263.786}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 246.934}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 229.062}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 304.237}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 437.435}]}
{"cur_time": "2024-08-20 15:05:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 382.086}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 379.961}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 344.955}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 493.154}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 376.686}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 309.125}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 414.397}]}
{"cur_time": "2024-08-20 15:06:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 459.715}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 406.35}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 236.628}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 267.348}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 369.368}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 249.32}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 286.307}]}
{"cur_time": "2024-08-20 15:07:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 412.961}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 311.701}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 381.652}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 288.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 357.541}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 322.484}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 297.858}]}
{"cur_time": "2024-08-20 15:08:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 449.789}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 374.23}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 334.691}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 261.363}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 386.219}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 417.254}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 320.172}]}
{"cur_time": "2024-08-20 15:09:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 434.203}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 336.344}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 310.148}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 292.62}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 361.84}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 323.022}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 272.127}]}
{"cur_time": "2024-08-20 15:10:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 339.329}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 368.253}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 435.63}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 324.063}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 384.532}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 303.32}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 436.045}]}
{"cur_time": "2024-08-20 15:11:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 444.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 361.683}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 264.886}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 289.969}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 246.695}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 385.325}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 310.359}]}
{"cur_time": "2024-08-20 15:12:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 394.4}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 295.434}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 436.926}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 404.886}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 336.799}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 350.937}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 374.555}]}
{"cur_time": "2024-08-20 15:13:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 427.629}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 293.474}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 428.425}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 302.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 397.014}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 294.319}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 260.654}]}
{"cur_time": "2024-08-20 15:14:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 376.515}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 413.098}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 374.428}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 431.259}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 403.821}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 308.834}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 235.096}]}
{"cur_time": "2024-08-20 15:15:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 432.569}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 304.802}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 357.588}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 256.427}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 408.089}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 303.228}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 403.586}]}
{"cur_time": "2024-08-20 15:16:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.8072509765625, "powerusage_W": 443.845}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.89947509765625, "powerusage_W": 231.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.51666259765625, "powerusage_W": 280.725}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.96978759765625, "powerusage_W": 282.331}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.97174072265625, "powerusage_W": 251.546}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.54595947265625, "powerusage_W": 234.011}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94244384765625, "powerusage_W": 258.913}]}
{"cur_time": "2024-08-20 15:17:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 54.24322509765625, "powerusage_W": 361.663}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.87646484375, "powerusage_W": 311.508}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.49365234375, "powerusage_W": 272.125}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94677734375, "powerusage_W": 373.904}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.94873046875, "powerusage_W": 256.182}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.84912109375, "powerusage_W": 296.616}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.42919921875, "powerusage_W": 338.237}]}
{"cur_time": "2024-08-20 15:18:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.67919921875, "powerusage_W": 433.413}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.85345458984375, "powerusage_W": 363.439}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.47064208984375, "powerusage_W": 366.953}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92376708984375, "powerusage_W": 425.181}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92572021484375, "powerusage_W": 375.764}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.15228271484375, "powerusage_W": 370.369}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.91595458984375, "powerusage_W": 280.646}]}
{"cur_time": "2024-08-20 15:19:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.67919921875, "powerusage_W": 425.961}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.85345458984375, "powerusage_W": 418.317}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.47064208984375, "powerusage_W": 399.892}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92376708984375, "powerusage_W": 351.702}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92572021484375, "powerusage_W": 419.158}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.15228271484375, "powerusage_W": 333.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.91595458984375, "powerusage_W": 422.529}]}
{"cur_time": "2024-08-20 15:20:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.67919921875, "powerusage_W": 389.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.85345458984375, "powerusage_W": 399.23}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.47064208984375, "powerusage_W": 342.655}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92376708984375, "powerusage_W": 240.703}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92572021484375, "powerusage_W": 304.132}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.15228271484375, "powerusage_W": 288.983}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.91595458984375, "powerusage_W": 304.329}]}
{"cur_time": "2024-08-20 15:21:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.67919921875, "powerusage_W": 423.574}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.85345458984375, "powerusage_W": 263.195}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.47064208984375, "powerusage_W": 240.776}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92376708984375, "powerusage_W": 331.929}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.92572021484375, "powerusage_W": 281.029}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.15228271484375, "powerusage_W": 344.56}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.91595458984375, "powerusage_W": 318.378}]}
{"cur_time": "2024-08-20 15:22:46", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.84368896484375, "powerusage_W": 323.605}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 90.862}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 95.768}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 87.935}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 87.623}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 89.433}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 87.288}]}
{"cur_time": "2024-08-20 15:23:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 22.10693359375, "powerusage_W": 320.845}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.943}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.826}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 87.037}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.95}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.623}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.196}]}
{"cur_time": "2024-08-20 15:24:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 463.592}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.654}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.905}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.582}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.528}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.112}]}
{"cur_time": "2024-08-20 15:25:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 417.438}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.571}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.426}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.404}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.519}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.959}]}
{"cur_time": "2024-08-20 15:26:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 441.606}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.387}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.663}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.341}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.412}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.918}]}
{"cur_time": "2024-08-20 15:27:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 425.947}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.38}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.481}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.662}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.316}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.37}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.894}]}
{"cur_time": "2024-08-20 15:28:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 452.029}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.31}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.411}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.647}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.266}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.325}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.079}]}
{"cur_time": "2024-08-20 15:29:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 425.587}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.335}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.412}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.626}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.399}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.238}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.089}]}
{"cur_time": "2024-08-20 15:30:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.90185546875, "powerusage_W": 433.796}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 88.298}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 94.411}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.632}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 85.405}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5902099609375, "powerusage_W": 88.294}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 86.086}]}
{"cur_time": "2024-08-20 15:31:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 495.429}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.395}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.604}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.262}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.111}]}
{"cur_time": "2024-08-20 15:32:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 488.55}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.329}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.457}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.594}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.425}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.261}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.11}]}
{"cur_time": "2024-08-20 15:33:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 488.942}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.33}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.397}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.578}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.289}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.132}]}
{"cur_time": "2024-08-20 15:34:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 485.166}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.238}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.338}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.559}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.234}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.27}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.09}]}
{"cur_time": "2024-08-20 15:35:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.65032958984375, "powerusage_W": 497.171}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.296}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.33}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.505}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.279}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.294}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.117}]}
{"cur_time": "2024-08-20 15:36:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 495.228}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.276}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.466}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.567}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.272}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.293}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.123}]}
{"cur_time": "2024-08-20 15:37:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 494.825}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.296}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.361}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.587}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.203}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.273}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.113}]}
{"cur_time": "2024-08-20 15:38:47", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 495.809}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.288}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.339}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.546}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.227}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.279}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.158}]}
{"cur_time": "2024-08-20 15:39:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 490.332}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.265}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.295}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.552}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.202}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.263}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.105}]}
{"cur_time": "2024-08-20 15:40:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 498.48}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.295}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.347}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.604}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.221}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.234}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.088}]}
{"cur_time": "2024-08-20 15:41:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 495.674}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.289}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.303}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.636}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.195}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.209}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.088}]}
{"cur_time": "2024-08-20 15:42:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 492.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.263}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.329}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.632}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.185}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.201}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.096}]}
{"cur_time": "2024-08-20 15:43:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 492.838}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.302}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.428}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.649}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.357}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.254}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.107}]}
{"cur_time": "2024-08-20 15:44:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 496.41}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.254}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.287}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.611}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.31}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.239}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.075}]}
{"cur_time": "2024-08-20 15:45:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 496.332}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.251}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.306}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.56}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.305}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.262}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}]}
{"cur_time": "2024-08-20 15:46:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.422}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.236}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.311}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.562}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.349}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.248}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.112}]}
{"cur_time": "2024-08-20 15:47:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 489.153}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.236}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.406}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.632}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.247}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.223}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.125}]}
{"cur_time": "2024-08-20 15:48:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 495.828}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.258}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.328}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.579}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.183}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.169}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.052}]}
{"cur_time": "2024-08-20 15:49:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 491.34}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.268}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.299}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.589}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.196}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.172}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.09}]}
{"cur_time": "2024-08-20 15:50:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 492.753}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.288}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.337}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.644}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.234}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.224}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.095}]}
{"cur_time": "2024-08-20 15:51:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 484.891}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.255}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.27}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.57}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.144}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.201}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 15:52:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.87}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.222}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.377}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.63}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.209}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.096}]}
{"cur_time": "2024-08-20 15:53:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 497.277}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.119}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.195}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.522}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.124}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.16}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.028}]}
{"cur_time": "2024-08-20 15:54:48", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.806}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.17}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.324}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.535}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.228}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.17}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 15:55:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 486.155}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.216}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.58}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.158}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.242}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.99}]}
{"cur_time": "2024-08-20 15:56:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.198}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.194}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.25}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.525}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.133}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.168}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.036}]}
{"cur_time": "2024-08-20 15:57:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 491.919}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.212}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.272}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.679}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.267}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 15:58:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 496.488}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.121}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.213}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.564}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.285}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.283}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.959}]}
{"cur_time": "2024-08-20 15:59:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.028}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.2}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.332}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.614}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.31}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.282}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.045}]}
{"cur_time": "2024-08-20 16:00:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 490.675}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.225}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.364}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.585}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.337}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.254}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.096}]}
{"cur_time": "2024-08-20 16:01:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.416}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.188}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.259}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.526}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.329}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.231}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}]}
{"cur_time": "2024-08-20 16:02:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.379}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.295}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.332}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.581}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.266}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.198}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.066}]}
{"cur_time": "2024-08-20 16:03:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.20111083984375, "powerusage_W": 487.747}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.18}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.271}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.291}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.254}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.007}]}
{"cur_time": "2024-08-20 16:04:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 489.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.177}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.257}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.276}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.03}]}
{"cur_time": "2024-08-20 16:05:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 488.58}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.166}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.284}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.513}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.221}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 16:06:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 490.035}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.192}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.407}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.534}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.158}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.151}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.043}]}
{"cur_time": "2024-08-20 16:07:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 490.369}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.238}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.343}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.195}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.119}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.078}]}
{"cur_time": "2024-08-20 16:08:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 495.843}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.314}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.442}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.591}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.332}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.22}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.071}]}
{"cur_time": "2024-08-20 16:09:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 496.201}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.221}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.4}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.567}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.336}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.204}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.105}]}
{"cur_time": "2024-08-20 16:10:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 493.022}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.21}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.582}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.285}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.215}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 16:11:49", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 493.641}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.133}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.286}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.523}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.286}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.119}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 16:12:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 493.104}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.141}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.315}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.547}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.264}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.121}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.011}]}
{"cur_time": "2024-08-20 16:13:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 496.572}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.222}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.378}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.586}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.285}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.226}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.037}]}
{"cur_time": "2024-08-20 16:14:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 486.108}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.255}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.322}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.533}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.298}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.176}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.049}]}
{"cur_time": "2024-08-20 16:15:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 484.841}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.277}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.552}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.147}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.951}]}
{"cur_time": "2024-08-20 16:16:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 496.899}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.196}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.359}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.503}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.282}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.138}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 16:17:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 489.385}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.214}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.432}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.612}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.256}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.126}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 16:18:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 492.164}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.206}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.305}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.597}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.165}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.058}]}
{"cur_time": "2024-08-20 16:19:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 489.797}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.194}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.338}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.626}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.156}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.268}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 16:20:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 497.368}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.227}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.546}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.089}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.285}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.99}]}
{"cur_time": "2024-08-20 16:21:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 490.11}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.159}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.303}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.56}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.188}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.042}]}
{"cur_time": "2024-08-20 16:22:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.379}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.17}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.365}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.51}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.177}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.004}]}
{"cur_time": "2024-08-20 16:23:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.889}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.217}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.398}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.581}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.208}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.089}]}
{"cur_time": "2024-08-20 16:24:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 490.535}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.161}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.304}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.127}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.167}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.089}]}
{"cur_time": "2024-08-20 16:25:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 495.795}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.273}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.521}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.145}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.283}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.982}]}
{"cur_time": "2024-08-20 16:26:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 488.529}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.185}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.325}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.607}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.172}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.013}]}
{"cur_time": "2024-08-20 16:27:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 426.358}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.168}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.334}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.582}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.298}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.192}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.043}]}
{"cur_time": "2024-08-20 16:28:50", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 484.628}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.151}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.309}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.612}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.204}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.074}]}
{"cur_time": "2024-08-20 16:29:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.694}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.225}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.311}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.577}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.294}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.211}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.932}]}
{"cur_time": "2024-08-20 16:30:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.675}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.153}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.27}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.582}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.234}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.187}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.962}]}
{"cur_time": "2024-08-20 16:31:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.337}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.186}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.365}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.555}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.305}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.259}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 16:32:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 485.04}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.258}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.362}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.572}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.286}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.036}]}
{"cur_time": "2024-08-20 16:33:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 491.205}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.206}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.339}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.569}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.137}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.236}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.088}]}
{"cur_time": "2024-08-20 16:34:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 495.785}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.217}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.372}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.139}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.179}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.005}]}
{"cur_time": "2024-08-20 16:35:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 493.131}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.275}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.396}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.578}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.209}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.929}]}
{"cur_time": "2024-08-20 16:36:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 487.373}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.196}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.338}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.652}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.297}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.211}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.991}]}
{"cur_time": "2024-08-20 16:37:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 492.953}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.214}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.387}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.661}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.365}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.283}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.039}]}
{"cur_time": "2024-08-20 16:38:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 492.091}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.286}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.393}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.333}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.291}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.043}]}
{"cur_time": "2024-08-20 16:39:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.5882568359375, "powerusage_W": 93.172}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.26}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.329}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.303}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.203}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.916}]}
{"cur_time": "2024-08-20 16:40:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.19134521484375, "powerusage_W": 495.938}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.071}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.286}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.559}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.309}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.163}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.967}]}
{"cur_time": "2024-08-20 16:41:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 494.704}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.291}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.591}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.297}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.21}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.945}]}
{"cur_time": "2024-08-20 16:42:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 490.81}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.233}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.384}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.608}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.184}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.216}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.999}]}
{"cur_time": "2024-08-20 16:43:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 492.984}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.284}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.536}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.122}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.194}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.931}]}
{"cur_time": "2024-08-20 16:44:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 496.675}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.178}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.248}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.593}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.118}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.223}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.036}]}
{"cur_time": "2024-08-20 16:45:51", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 477.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.102}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.211}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.592}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.244}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.973}]}
{"cur_time": "2024-08-20 16:46:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 493.033}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.178}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.329}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.637}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.23}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.25}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.015}]}
{"cur_time": "2024-08-20 16:47:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 495.848}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.142}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.22}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.487}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.292}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.185}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.969}]}
{"cur_time": "2024-08-20 16:48:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64642333984375, "powerusage_W": 494.7}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.096}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.281}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.562}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.272}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.183}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.005}]}
{"cur_time": "2024-08-20 16:49:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.64837646484375, "powerusage_W": 494.374}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.294}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.515}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.285}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.129}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.011}]}
{"cur_time": "2024-08-20 16:50:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 490.913}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.174}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.429}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.6}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.297}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.232}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.051}]}
{"cur_time": "2024-08-20 16:51:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 490.256}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.322}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.517}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.329}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.24}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.097}]}
{"cur_time": "2024-08-20 16:52:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.415}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.234}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.543}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.304}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.23}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.951}]}
{"cur_time": "2024-08-20 16:53:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 495.735}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.45}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.117}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.982}]}
{"cur_time": "2024-08-20 16:54:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 495.624}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.116}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.292}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.477}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.258}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.161}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.038}]}
{"cur_time": "2024-08-20 16:55:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 493.857}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.085}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.321}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.5}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.275}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.181}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.05}]}
{"cur_time": "2024-08-20 16:56:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 493.704}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.172}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.215}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.598}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.249}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.013}]}
{"cur_time": "2024-08-20 16:57:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 493.3}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.276}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.552}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.24}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.158}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.0}]}
{"cur_time": "2024-08-20 16:58:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 495.868}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.09}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.269}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.53}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.234}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.186}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.954}]}
{"cur_time": "2024-08-20 16:59:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 496.092}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.112}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.265}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.53}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.215}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.115}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.999}]}
{"cur_time": "2024-08-20 17:00:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 489.532}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.116}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.224}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.185}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.887}]}
{"cur_time": "2024-08-20 17:01:52", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 494.226}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.142}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.243}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.557}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.094}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.197}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.933}]}
{"cur_time": "2024-08-20 17:02:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 491.815}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.139}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.286}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.538}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.118}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.147}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.89}]}
{"cur_time": "2024-08-20 17:03:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 496.353}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.138}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.172}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.511}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.153}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.838}]}
{"cur_time": "2024-08-20 17:04:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 484.588}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.183}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.488}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.183}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.89}]}
{"cur_time": "2024-08-20 17:05:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 493.654}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.276}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.546}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.051}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.156}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.059}]}
{"cur_time": "2024-08-20 17:06:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 496.603}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.165}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.306}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.581}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.15}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.196}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.096}]}
{"cur_time": "2024-08-20 17:07:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13665771484375, "powerusage_W": 497.131}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.07}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.198}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.463}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.188}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.909}]}
{"cur_time": "2024-08-20 17:08:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 492.128}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.029}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.212}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.514}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.174}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.959}]}
{"cur_time": "2024-08-20 17:09:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.774}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.136}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.235}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.502}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.146}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.962}]}
{"cur_time": "2024-08-20 17:10:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 494.334}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.26}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.491}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.12}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.932}]}
{"cur_time": "2024-08-20 17:11:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 482.374}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.096}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.214}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.484}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.917}]}
{"cur_time": "2024-08-20 17:12:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 494.44}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.061}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.031}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.008}]}
{"cur_time": "2024-08-20 17:13:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 494.795}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.204}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.575}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.227}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.134}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.959}]}
{"cur_time": "2024-08-20 17:14:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 496.014}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.193}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.182}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.137}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.857}]}
{"cur_time": "2024-08-20 17:15:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 19.13861083984375, "powerusage_W": 493.537}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.019}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.207}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.53}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.201}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.876}]}
{"cur_time": "2024-08-20 17:16:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.034}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.135}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.19}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.122}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.851}]}
{"cur_time": "2024-08-20 17:17:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.494}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.138}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.161}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.524}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.188}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.149}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.939}]}
{"cur_time": "2024-08-20 17:18:53", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.396}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.982}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.152}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.549}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.127}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.879}]}
{"cur_time": "2024-08-20 17:19:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.346}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.231}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.18}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.87}]}
{"cur_time": "2024-08-20 17:20:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.365}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.949}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 17:21:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.94}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.213}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.124}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.792}]}
{"cur_time": "2024-08-20 17:22:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.013}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.479}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 17:23:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.988}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.244}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.498}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.168}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.845}]}
{"cur_time": "2024-08-20 17:24:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.366}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.156}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.821}]}
{"cur_time": "2024-08-20 17:25:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.98}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.244}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.193}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.824}]}
{"cur_time": "2024-08-20 17:26:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.305}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.933}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.422}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.219}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.848}]}
{"cur_time": "2024-08-20 17:27:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.978}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.213}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.502}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.158}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.108}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.87}]}
{"cur_time": "2024-08-20 17:28:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.219}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.154}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.482}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.005}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.065}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.833}]}
{"cur_time": "2024-08-20 17:29:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.964}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.063}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.011}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.134}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.833}]}
{"cur_time": "2024-08-20 17:30:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.323}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.953}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.167}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.437}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.018}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.139}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.833}]}
{"cur_time": "2024-08-20 17:31:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.272}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.064}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.415}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.071}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.149}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.9}]}
{"cur_time": "2024-08-20 17:32:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.306}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.937}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.161}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.476}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.118}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.127}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.9}]}
{"cur_time": "2024-08-20 17:33:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.329}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.037}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.496}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.047}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.222}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.907}]}
{"cur_time": "2024-08-20 17:34:54", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.361}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.263}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.546}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.177}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.193}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.908}]}
{"cur_time": "2024-08-20 17:35:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.009}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.163}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.204}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.938}]}
{"cur_time": "2024-08-20 17:36:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.356}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.018}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.142}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.521}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.185}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.172}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.875}]}
{"cur_time": "2024-08-20 17:37:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.393}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.107}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.241}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.586}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.207}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.185}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.952}]}
{"cur_time": "2024-08-20 17:38:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.318}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.149}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.527}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.938}]}
{"cur_time": "2024-08-20 17:39:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.259}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.942}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.458}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.194}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.155}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.969}]}
{"cur_time": "2024-08-20 17:40:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.323}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.011}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.174}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.469}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.147}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.204}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.948}]}
{"cur_time": "2024-08-20 17:41:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.367}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.954}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.466}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.121}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.125}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.949}]}
{"cur_time": "2024-08-20 17:42:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.348}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.02}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.228}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.491}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.173}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.122}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.958}]}
{"cur_time": "2024-08-20 17:43:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.371}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.009}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.516}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.2}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.173}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.971}]}
{"cur_time": "2024-08-20 17:44:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.261}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.167}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.122}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.925}]}
{"cur_time": "2024-08-20 17:45:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.355}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.229}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.453}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.194}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.173}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.899}]}
{"cur_time": "2024-08-20 17:46:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.336}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.01}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.46}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.115}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.905}]}
{"cur_time": "2024-08-20 17:47:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.342}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.011}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.221}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.548}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.275}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.169}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.014}]}
{"cur_time": "2024-08-20 17:48:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.424}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.05}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.319}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.558}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.176}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.162}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.889}]}
{"cur_time": "2024-08-20 17:49:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.378}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.016}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.229}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.537}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.009}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.162}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.914}]}
{"cur_time": "2024-08-20 17:50:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.954}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.006}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.944}]}
{"cur_time": "2024-08-20 17:51:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.374}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.208}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.012}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.912}]}
{"cur_time": "2024-08-20 17:52:55", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.278}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.137}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.437}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.992}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.05}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.901}]}
{"cur_time": "2024-08-20 17:53:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.124}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.439}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.088}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.198}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.974}]}
{"cur_time": "2024-08-20 17:54:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.387}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.963}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.491}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.974}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.159}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.921}]}
{"cur_time": "2024-08-20 17:55:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.896}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.208}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.463}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.162}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.094}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.916}]}
{"cur_time": "2024-08-20 17:56:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.336}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.991}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.221}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.498}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.154}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.893}]}
{"cur_time": "2024-08-20 17:57:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.942}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.232}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.931}]}
{"cur_time": "2024-08-20 17:58:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.272}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.948}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.157}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.178}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.937}]}
{"cur_time": "2024-08-20 17:59:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.323}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.01}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.496}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.131}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.164}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.894}]}
{"cur_time": "2024-08-20 18:00:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.274}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.965}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.134}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.126}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.821}]}
{"cur_time": "2024-08-20 18:01:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.235}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.212}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.469}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.202}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.101}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.856}]}
{"cur_time": "2024-08-20 18:02:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.371}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.981}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.166}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.128}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.142}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.825}]}
{"cur_time": "2024-08-20 18:03:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.137}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.741}]}
{"cur_time": "2024-08-20 18:04:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.976}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.215}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:05:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.991}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.065}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 18:06:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.325}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.079}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.058}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:07:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.357}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.986}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.194}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.5}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.15}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.146}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 18:08:56", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.31}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.199}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.546}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.167}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.102}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.871}]}
{"cur_time": "2024-08-20 18:09:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.173}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.464}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.135}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.833}]}
{"cur_time": "2024-08-20 18:10:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.91}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.141}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 18:11:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.223}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 18:12:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.987}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.164}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.126}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 18:13:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.981}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.251}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 18:14:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.315}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.974}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.122}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 18:15:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.244}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.001}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.137}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.857}]}
{"cur_time": "2024-08-20 18:16:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.952}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.162}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.133}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:17:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.963}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.24}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.416}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.027}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-20 18:18:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.338}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.934}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.14}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.132}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 18:19:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.373}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.941}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.994}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.849}]}
{"cur_time": "2024-08-20 18:20:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.97}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.008}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.849}]}
{"cur_time": "2024-08-20 18:21:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.239}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.951}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.162}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.761}]}
{"cur_time": "2024-08-20 18:22:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.928}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.123}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.937}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 18:23:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.377}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.975}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.171}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.513}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.101}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 18:24:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 18:25:57", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.25}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.884}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.098}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.054}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:26:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.256}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.851}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.077}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.059}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.731}]}
{"cur_time": "2024-08-20 18:27:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.179}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.46}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.114}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.098}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.778}]}
{"cur_time": "2024-08-20 18:28:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.98}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.111}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-20 18:29:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.253}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.739}]}
{"cur_time": "2024-08-20 18:30:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.281}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:31:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.949}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.072}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 18:32:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.856}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.081}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.069}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.096}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 18:33:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.171}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 18:34:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.352}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.92}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.176}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.473}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.946}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 18:35:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.204}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.081}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 18:36:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.158}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 18:37:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.256}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.168}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.071}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 18:38:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.02}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.227}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.488}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 18:39:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.197}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.897}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.165}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.051}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:40:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.315}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.905}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.117}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.827}]}
{"cur_time": "2024-08-20 18:41:58", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.269}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.819}]}
{"cur_time": "2024-08-20 18:42:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.932}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.104}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.061}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 18:43:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.343}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.928}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.179}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.121}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.812}]}
{"cur_time": "2024-08-20 18:44:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.203}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.9}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.124}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.406}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.984}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.818}]}
{"cur_time": "2024-08-20 18:45:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.015}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.049}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:46:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.982}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:47:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.821}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.012}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.034}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 18:48:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.847}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.117}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.116}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:49:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.201}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 18:50:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.945}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.123}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:51:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:52:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.241}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.887}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.432}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 18:53:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 18:54:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.273}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.911}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.066}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 18:55:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 18:56:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.324}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.937}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.066}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 18:57:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.243}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.947}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.088}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.991}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 18:58:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.208}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.905}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.055}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 18:59:59", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.199}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.881}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.411}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-20 19:01:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.241}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.835}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.035}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.379}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 19:02:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.229}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.878}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.854}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 19:03:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.976}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.198}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:04:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.937}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 19:05:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.182}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.072}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 19:06:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.244}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.868}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.438}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.068}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:07:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.237}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.95}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.112}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:08:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.903}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.128}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 19:09:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.887}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 19:10:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.972}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.446}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 19:11:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.339}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.894}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.107}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 19:12:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.268}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.077}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:13:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.145}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.981}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 19:14:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.934}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.416}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.745}]}
{"cur_time": "2024-08-20 19:15:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.87}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.176}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:16:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.92}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.101}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:17:00", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.924}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.149}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:18:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.326}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.976}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.104}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.981}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.108}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:19:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.333}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.971}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.151}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:20:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:21:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.932}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:22:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.126}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.089}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 19:23:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.325}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.949}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.827}]}
{"cur_time": "2024-08-20 19:24:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.278}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.094}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-20 19:25:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 19:26:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.227}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.944}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.102}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.097}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 19:27:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.115}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 19:28:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.209}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.947}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.131}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.006}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.737}]}
{"cur_time": "2024-08-20 19:29:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.239}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.835}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.028}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 19:30:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.88}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.048}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 19:31:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.3}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.905}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.127}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.929}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.063}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 19:32:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.933}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.088}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 19:33:01", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.116}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:34:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.317}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.947}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.434}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.079}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 19:35:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.9}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.07}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:36:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.135}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 19:37:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.29}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.105}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.035}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:38:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.176}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.085}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 19:39:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.924}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.087}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 19:40:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.29}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.123}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.916}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:41:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.309}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.968}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.127}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.867}]}
{"cur_time": "2024-08-20 19:42:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.214}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.844}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.112}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.917}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:43:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.269}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.89}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:44:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.036}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:45:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.925}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 19:46:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.241}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.048}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:47:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.243}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.884}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.084}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.359}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:48:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.326}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.002}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.166}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.133}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-20 19:49:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.878}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.055}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 19:50:02", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.243}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.949}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.072}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 19:51:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.239}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.914}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.073}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 19:52:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.886}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.107}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 19:53:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.878}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.805}]}
{"cur_time": "2024-08-20 19:54:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.279}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.18}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.025}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:55:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.949}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 19:56:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.943}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.842}]}
{"cur_time": "2024-08-20 19:57:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.915}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 19:58:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.238}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.821}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.918}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.327}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.038}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.717}]}
{"cur_time": "2024-08-20 19:59:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.04}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-20 20:00:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.246}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.014}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.042}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:01:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 20:02:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.242}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.915}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.083}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.044}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:03:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.94}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.069}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.058}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 20:04:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.914}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:05:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.29}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.95}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.109}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:06:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.184}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.816}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.964}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:07:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.334}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.963}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.981}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:08:03", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.34}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.948}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.874}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.416}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.083}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 20:09:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.214}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.074}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:10:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.974}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.088}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.851}]}
{"cur_time": "2024-08-20 20:11:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.914}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.068}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.059}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.083}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 20:12:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.275}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.115}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.033}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 20:13:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.332}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.957}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.044}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 20:14:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.97}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.146}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 20:15:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.312}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.962}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 20:16:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.132}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.029}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.76}]}
{"cur_time": "2024-08-20 20:17:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.285}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.898}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.133}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 20:18:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.091}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.753}]}
{"cur_time": "2024-08-20 20:19:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.274}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.871}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.028}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:20:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.309}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.96}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.939}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 20:21:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.965}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.131}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.121}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:22:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.278}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.107}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 20:23:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.129}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 20:24:04", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.31}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:25:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.962}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.146}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.076}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-20 20:26:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.961}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 20:27:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.284}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.46}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 20:28:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.295}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:29:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.982}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.776}]}
{"cur_time": "2024-08-20 20:30:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.831}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.038}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.327}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.87}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.064}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 20:31:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.847}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.89}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.03}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.722}]}
{"cur_time": "2024-08-20 20:32:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.124}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 20:33:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.89}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.031}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.049}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 20:34:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.278}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.143}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 20:35:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.261}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.862}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.046}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.453}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.041}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:36:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.879}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.982}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 20:37:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.328}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.084}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.088}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.009}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 20:38:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.2}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.821}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.958}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.334}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.013}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.063}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 20:39:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.266}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 20:40:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.316}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.859}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.074}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:41:05", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.139}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.854}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.021}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.074}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 20:42:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.934}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.05}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 20:43:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.826}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.069}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.065}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:44:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.205}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.78}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.884}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:45:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.219}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.106}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:46:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.029}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 20:47:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.231}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.83}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.916}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 20:48:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:49:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.311}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.842}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.459}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.054}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 20:50:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.963}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.144}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 20:51:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.137}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.797}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.92}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.309}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.995}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.022}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 20:52:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.199}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.812}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.042}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.033}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.044}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.73}]}
{"cur_time": "2024-08-20 20:53:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.253}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.833}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.989}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.042}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 20:54:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.83}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.048}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.917}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.107}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 20:55:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.292}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.074}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 20:56:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.208}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.879}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.995}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.878}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 20:57:06", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.304}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.889}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.128}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 20:58:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.943}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.069}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.37}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.935}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 20:59:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.876}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.033}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:00:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.846}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.036}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:01:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.104}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.469}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.09}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.743}]}
{"cur_time": "2024-08-20 21:02:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.305}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.051}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 21:03:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.012}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.038}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.057}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 21:04:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.204}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.857}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 21:05:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.226}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.845}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.061}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.039}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:06:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.169}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.027}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.316}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 21:07:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.183}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.809}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.914}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.02}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.737}]}
{"cur_time": "2024-08-20 21:08:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.267}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.028}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:09:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.242}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.954}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.107}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 21:10:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.275}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.968}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.12}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-20 21:11:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.201}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.042}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.396}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:12:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.174}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.875}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.984}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.359}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.014}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.745}]}
{"cur_time": "2024-08-20 21:13:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.169}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.016}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:14:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.816}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.986}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.006}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.73}]}
{"cur_time": "2024-08-20 21:15:07", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.229}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.063}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.046}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:16:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.227}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.849}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.993}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.728}]}
{"cur_time": "2024-08-20 21:17:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.157}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.856}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.341}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.083}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:18:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.806}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.98}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.333}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.903}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.722}]}
{"cur_time": "2024-08-20 21:19:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.905}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.018}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.057}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 21:20:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.909}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.101}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.097}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.745}]}
{"cur_time": "2024-08-20 21:21:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.21}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.794}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.098}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.897}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.038}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:22:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.823}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.076}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.897}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.045}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 21:23:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 21:24:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.175}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.913}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.033}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:25:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.25}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.87}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.11}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 21:26:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.299}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.893}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.076}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.038}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.054}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 21:27:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.895}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.051}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.038}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.042}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:28:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.148}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.893}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.038}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.982}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.018}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:29:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.158}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.072}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:30:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.129}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.854}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.025}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.407}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.89}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.003}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:31:08", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.86}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.975}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.04}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:32:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.218}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.89}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.063}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.427}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 21:33:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.198}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.391}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.903}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.043}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.722}]}
{"cur_time": "2024-08-20 21:34:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.186}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.01}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.365}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.023}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:35:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.184}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.827}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.045}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.365}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.903}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:36:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.204}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.758}]}
{"cur_time": "2024-08-20 21:37:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.216}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.078}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.36}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.055}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:38:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.167}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.805}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.901}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.295}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.883}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.979}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.728}]}
{"cur_time": "2024-08-20 21:39:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.844}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.028}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:40:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.171}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.779}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.001}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.001}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 21:41:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.04}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.031}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 21:42:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.204}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.055}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.05}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 21:43:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.208}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.972}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.006}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.029}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.73}]}
{"cur_time": "2024-08-20 21:44:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.254}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.873}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.948}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:45:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.246}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.881}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.084}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 21:46:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.239}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.094}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.047}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.821}]}
{"cur_time": "2024-08-20 21:47:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.124}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.848}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.038}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.334}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:48:09", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.207}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.851}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.065}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.031}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.006}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 21:49:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.197}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.831}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.015}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.112}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.728}]}
{"cur_time": "2024-08-20 21:50:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.158}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.944}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:51:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.213}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.886}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.076}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.377}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.02}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 21:52:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.834}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.048}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.916}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:53:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.226}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.916}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.055}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.737}]}
{"cur_time": "2024-08-20 21:54:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.182}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.886}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.958}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.016}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 21:55:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.256}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.895}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.101}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.397}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.028}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 21:56:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.204}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.903}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.05}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.366}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.046}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:57:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.178}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.384}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.116}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 21:58:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.178}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.989}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.333}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 21:59:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.833}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.019}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.027}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.027}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 22:00:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.197}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.799}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.379}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.008}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.057}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 22:01:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.82}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.031}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.34}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.013}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.1}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 22:02:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.125}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.795}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.951}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.294}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.008}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.04}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.722}]}
{"cur_time": "2024-08-20 22:03:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.194}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.875}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.046}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.878}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.049}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 22:04:10", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.242}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.846}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.071}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 22:05:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.817}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.057}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.917}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:06:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.968}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 22:07:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.899}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.016}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:08:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.098}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 22:09:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.106}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.102}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:10:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.843}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.085}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.084}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 22:11:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.268}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.924}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.076}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.745}]}
{"cur_time": "2024-08-20 22:12:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.171}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.074}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.91}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.015}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:13:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.846}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.016}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.903}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.074}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:14:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.882}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.088}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.416}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 22:15:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.835}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.023}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.014}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.071}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 22:16:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.886}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.016}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.379}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.051}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 22:17:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.206}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.933}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.056}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.73}]}
{"cur_time": "2024-08-20 22:18:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.169}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.842}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.036}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.379}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 22:19:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.21}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.86}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.114}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 22:20:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.902}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.148}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.773}]}
{"cur_time": "2024-08-20 22:21:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.918}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.068}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 22:22:11", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.941}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.142}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.076}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 22:23:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.888}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.029}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:24:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.954}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.039}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:25:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.195}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.861}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.998}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 22:26:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.254}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.853}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.89}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 22:27:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.19}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.839}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.051}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.981}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.136}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 22:28:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.024}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 22:29:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.152}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.974}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.09}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 22:30:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.281}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.944}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.11}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.087}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 22:31:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.338}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 22:32:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.198}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.895}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.076}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.069}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.101}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 22:33:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.171}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.826}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.031}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.033}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.094}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:34:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.897}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.093}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 22:35:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.263}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.897}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.085}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.398}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.032}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.049}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 22:36:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.228}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.884}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.077}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 22:37:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.272}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.895}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.128}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.033}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 22:38:12", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.195}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.886}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.99}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.119}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:39:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.218}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.901}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:40:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.829}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.031}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.366}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:41:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.188}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.91}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.036}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.353}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.087}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 22:42:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.931}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.804}]}
{"cur_time": "2024-08-20 22:43:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.201}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.825}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.949}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.371}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.897}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.102}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.736}]}
{"cur_time": "2024-08-20 22:44:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.842}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.021}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.017}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 22:45:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.265}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.907}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.093}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.385}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 22:46:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.275}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.094}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 22:47:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.838}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.09}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.937}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.066}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 22:48:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.32}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.903}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 22:49:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.87}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.107}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.955}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.044}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 22:50:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.296}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.896}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-20 22:51:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.271}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.079}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.416}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:52:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.268}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.952}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.053}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.049}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.73}]}
{"cur_time": "2024-08-20 22:53:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.236}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.095}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.052}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.783}]}
{"cur_time": "2024-08-20 22:54:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.28}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.89}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.105}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 22:55:13", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.079}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 22:56:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.23}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.93}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.986}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.014}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.737}]}
{"cur_time": "2024-08-20 22:57:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.224}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.126}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 22:58:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.178}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.88}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.973}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.352}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.95}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 22:59:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.289}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.877}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.14}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.072}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 23:00:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.319}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.087}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.942}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.129}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 23:01:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.912}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 23:02:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.428}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.091}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 23:03:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.259}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.11}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.059}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 23:04:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.276}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.144}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.415}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.027}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 23:05:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.906}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.111}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.365}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.93}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.135}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:06:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.896}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.109}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.974}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.069}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 23:07:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.255}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.082}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.073}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.744}]}
{"cur_time": "2024-08-20 23:08:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.296}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.967}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.442}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.982}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 23:09:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.254}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.938}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.125}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.844}]}
{"cur_time": "2024-08-20 23:10:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.296}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.968}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.751}]}
{"cur_time": "2024-08-20 23:11:14", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.207}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.039}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.078}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 23:12:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.358}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.991}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.507}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-20 23:13:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.883}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.082}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.074}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:14:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.341}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.904}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.069}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.085}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.729}]}
{"cur_time": "2024-08-20 23:15:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.896}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.168}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.046}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.09}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:16:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.146}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.923}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.061}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 23:17:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.909}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.161}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 23:18:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.164}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.107}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 23:19:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.333}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.035}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.152}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.114}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.097}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.851}]}
{"cur_time": "2024-08-20 23:20:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.29}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.915}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.116}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.12}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.038}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.752}]}
{"cur_time": "2024-08-20 23:21:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.309}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.953}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.12}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.066}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:22:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.981}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.079}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:23:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.275}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.943}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.127}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.075}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.789}]}
{"cur_time": "2024-08-20 23:24:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.106}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.429}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 23:25:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.268}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.16}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.449}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.962}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.078}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 23:26:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.876}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.17}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.098}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:27:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.286}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.925}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.168}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.006}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.068}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-20 23:28:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.34}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.902}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.133}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.056}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.844}]}
{"cur_time": "2024-08-20 23:29:15", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.281}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.831}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.022}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.076}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.048}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 23:30:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.206}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.856}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.046}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.768}]}
{"cur_time": "2024-08-20 23:31:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.305}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.933}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.144}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.045}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.819}]}
{"cur_time": "2024-08-20 23:32:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.334}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.965}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.134}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.082}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-20 23:33:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.328}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.99}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.13}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.124}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.774}]}
{"cur_time": "2024-08-20 23:34:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.999}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.126}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.111}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 23:35:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.306}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.944}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.191}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.152}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.068}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.811}]}
{"cur_time": "2024-08-20 23:36:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.966}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.893}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-20 23:37:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.266}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.155}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 23:38:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.097}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.085}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.104}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.805}]}
{"cur_time": "2024-08-20 23:39:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.333}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.944}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.167}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.905}]}
{"cur_time": "2024-08-20 23:40:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.318}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.882}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.143}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.821}]}
{"cur_time": "2024-08-20 23:41:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.262}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.959}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.154}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.475}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.1}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-20 23:42:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.227}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.017}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.378}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.07}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-20 23:43:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.269}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.149}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.461}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.098}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.082}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-20 23:44:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.286}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.108}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.092}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 23:45:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.976}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.127}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-20 23:46:16", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.165}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.956}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.103}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.806}]}
{"cur_time": "2024-08-20 23:47:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.258}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.95}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.089}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.43}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 23:48:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.249}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.932}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.129}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.359}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.79}]}
{"cur_time": "2024-08-20 23:49:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.252}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.872}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.1}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.936}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.032}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.805}]}
{"cur_time": "2024-08-20 23:50:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.942}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.969}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 23:51:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.36}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.136}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.133}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.087}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.805}]}
{"cur_time": "2024-08-20 23:52:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.308}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.909}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.454}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.137}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.044}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-20 23:53:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.288}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.919}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.119}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.417}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.111}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.782}]}
{"cur_time": "2024-08-20 23:54:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.217}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.876}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.046}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.347}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.759}]}
{"cur_time": "2024-08-20 23:55:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.946}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.145}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.101}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.106}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.775}]}
{"cur_time": "2024-08-20 23:56:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.975}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.475}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-20 23:57:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.301}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.956}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.121}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.131}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.834}]}
{"cur_time": "2024-08-20 23:58:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.975}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.143}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.123}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.849}]}
{"cur_time": "2024-08-20 23:59:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.243}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.924}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.19}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.949}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.095}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.753}]}
{"cur_time": "2024-08-21 00:00:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.385}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.017}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.513}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.019}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.131}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.894}]}
{"cur_time": "2024-08-21 00:01:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.379}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.961}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.462}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.007}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.056}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-21 00:02:17", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.297}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.945}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.186}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.767}]}
{"cur_time": "2024-08-21 00:03:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.294}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.225}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.095}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-21 00:04:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.322}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.944}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.423}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.092}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.796}]}
{"cur_time": "2024-08-21 00:05:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.333}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.99}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.125}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.117}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-21 00:06:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.312}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.967}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.157}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.475}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.968}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-21 00:07:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.277}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.958}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.152}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.943}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.099}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.805}]}
{"cur_time": "2024-08-21 00:08:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.295}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.963}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.151}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.001}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.111}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.859}]}
{"cur_time": "2024-08-21 00:09:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.314}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.927}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.436}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.975}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-21 00:10:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.34}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.474}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.031}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.119}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.851}]}
{"cur_time": "2024-08-21 00:11:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.297}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.973}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.167}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.435}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.987}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.168}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.819}]}
{"cur_time": "2024-08-21 00:12:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.328}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.943}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.505}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.995}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.179}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.844}]}
{"cur_time": "2024-08-21 00:13:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.366}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.997}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.153}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.501}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.132}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.836}]}
{"cur_time": "2024-08-21 00:14:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.353}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.999}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.214}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.443}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.145}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.153}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.821}]}
{"cur_time": "2024-08-21 00:15:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.964}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.236}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.828}]}
{"cur_time": "2024-08-21 00:16:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.307}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.94}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.455}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.126}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.781}]}
{"cur_time": "2024-08-21 00:17:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.979}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.142}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.456}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.156}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.136}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-21 00:18:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.372}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.089}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.176}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.513}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.201}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.99}]}
{"cur_time": "2024-08-21 00:19:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.354}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.955}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.864}]}
{"cur_time": "2024-08-21 00:20:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.333}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.058}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.152}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.007}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.109}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.813}]}
{"cur_time": "2024-08-21 00:21:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.386}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.983}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.242}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.48}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.995}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.175}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.826}]}
{"cur_time": "2024-08-21 00:22:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.995}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.214}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.473}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.995}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.797}]}
{"cur_time": "2024-08-21 00:23:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.385}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.053}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.192}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.019}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.188}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.844}]}
{"cur_time": "2024-08-21 00:24:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.385}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.042}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.25}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.511}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.028}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.175}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.858}]}
{"cur_time": "2024-08-21 00:25:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.372}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.023}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.988}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.82}]}
{"cur_time": "2024-08-21 00:26:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.386}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.041}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.224}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.524}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.025}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.205}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.819}]}
{"cur_time": "2024-08-21 00:27:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.418}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.067}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.183}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.543}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.237}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.944}]}
{"cur_time": "2024-08-21 00:28:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.4}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.948}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.131}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.856}]}
{"cur_time": "2024-08-21 00:29:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.386}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.062}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.207}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.487}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.141}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.173}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.841}]}
{"cur_time": "2024-08-21 00:30:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.418}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.028}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.191}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.519}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.148}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.134}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.841}]}
{"cur_time": "2024-08-21 00:31:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.429}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.054}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.263}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.545}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.187}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.171}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.84}]}
{"cur_time": "2024-08-21 00:32:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.366}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.992}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.251}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.14}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.841}]}
{"cur_time": "2024-08-21 00:33:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.419}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.989}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.555}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.848}]}
{"cur_time": "2024-08-21 00:34:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.353}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.001}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.231}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.448}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.148}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.852}]}
{"cur_time": "2024-08-21 00:35:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.45}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.258}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.533}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.038}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.181}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.943}]}
{"cur_time": "2024-08-21 00:36:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.379}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.06}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.244}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.012}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.114}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.82}]}
{"cur_time": "2024-08-21 00:37:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.379}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.006}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.249}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.995}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.894}]}
{"cur_time": "2024-08-21 00:38:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.371}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.98}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.225}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.063}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.148}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.91}]}
{"cur_time": "2024-08-21 00:39:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.403}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.078}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.179}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.544}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.043}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.19}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.864}]}
{"cur_time": "2024-08-21 00:40:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.346}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.017}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.238}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.498}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.205}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.93}]}
{"cur_time": "2024-08-21 00:41:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.341}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.036}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.251}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.467}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.164}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.124}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.934}]}
{"cur_time": "2024-08-21 00:42:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.399}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.982}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.168}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.523}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.199}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.162}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.848}]}
{"cur_time": "2024-08-21 00:43:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.359}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.991}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.219}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.468}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.141}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.16}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.824}]}
{"cur_time": "2024-08-21 00:44:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.366}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.023}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.242}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.494}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.15}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.819}]}
{"cur_time": "2024-08-21 00:45:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.303}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.928}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.105}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.346}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.079}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.766}]}
{"cur_time": "2024-08-21 00:46:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.916}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.203}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.486}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.12}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.13}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.857}]}
{"cur_time": "2024-08-21 00:47:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.313}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.994}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.131}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.5}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.096}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.143}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.825}]}
{"cur_time": "2024-08-21 00:48:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.238}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.922}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.084}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.41}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.115}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.117}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.817}]}
{"cur_time": "2024-08-21 00:49:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.247}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.875}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.032}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.405}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.138}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.12}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.887}]}
{"cur_time": "2024-08-21 00:50:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.221}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.885}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.045}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.361}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.057}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.101}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.806}]}
{"cur_time": "2024-08-21 00:51:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.196}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.842}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.026}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.32}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.037}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.0}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.754}]}
{"cur_time": "2024-08-21 00:52:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.186}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.794}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.96}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.406}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.056}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.027}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.818}]}
{"cur_time": "2024-08-21 00:53:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.177}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.829}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.015}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.317}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.011}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 88.03}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.798}]}
{"cur_time": "2024-08-21 00:54:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.17}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.018}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.262}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.011}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.984}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.753}]}
{"cur_time": "2024-08-21 00:55:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.179}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.718}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.948}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.308}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.986}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.961}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.672}]}
{"cur_time": "2024-08-21 00:56:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.079}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.732}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.936}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.315}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.026}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.931}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.704}]}
{"cur_time": "2024-08-21 00:57:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.135}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.746}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.02}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.344}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.915}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.965}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.662}]}
{"cur_time": "2024-08-21 00:58:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.138}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.841}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.097}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.338}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.803}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.924}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.7}]}
{"cur_time": "2024-08-21 00:59:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.064}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.755}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 94.05}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.287}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.935}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.666}]}
{"cur_time": "2024-08-21 01:00:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.109}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.767}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.989}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.306}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.813}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.961}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.642}]}
{"cur_time": "2024-08-21 01:01:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.019}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.625}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.909}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.247}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.787}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.9}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.609}]}
{"cur_time": "2024-08-21 01:02:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 91.929}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.645}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.834}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.191}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.69}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.881}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.494}]}
{"cur_time": "2024-08-21 01:03:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 92.069}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.673}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.872}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.237}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.711}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.863}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.563}]}
{"cur_time": "2024-08-21 01:04:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 91.951}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.59}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.861}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.153}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.65}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.76}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.502}]}
{"cur_time": "2024-08-21 01:05:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 91.86}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.554}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 93.863}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 86.082}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 84.655}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 87.8}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 0.58502197265625, "powerusage_W": 85.501}]}
Reference in New Issue View Git Blame Copy Permalink