p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

train: add inference results

This commit is contained in:
wql 2024-08-26 06:44:32 +00:00
parent 51faa7dda7
commit f45e96825b
88 changed files with 2782 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
results/gpu_status_0826.json Normal file
View File

@ -0,0 +1,151 @@
{"cur_time": "2024-08-26 04:11:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.88}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.042}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.503}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.403}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.159}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.059}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.2}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:12:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.244}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 138.612}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.965}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.75}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.61}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 134.688}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:13:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.743}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.084}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.465}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.147}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.547}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.537}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:14:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.289}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 139.57}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.056}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.081}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.097}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.033}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.203}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:15:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.75341796875, "powerusage_W": 140.352}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.09002685546875, "powerusage_W": 138.861}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.0201416015625, "powerusage_W": 139.895}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6490478515625, "powerusage_W": 135.639}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.1920166015625, "powerusage_W": 130.611}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.0201416015625, "powerusage_W": 135.445}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.5513916015625, "powerusage_W": 133.725}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 13.447265625}]}
{"cur_time": "2024-08-26 04:16:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.58935546875, "powerusage_W": 341.646}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.20916748046875, "powerusage_W": 336.278}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3970947265625, "powerusage_W": 337.97}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 332.031}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3111572265625, "powerusage_W": 323.555}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3931884765625, "powerusage_W": 324.689}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.9224853515625, "powerusage_W": 325.899}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3533197, "create_time": 1724645550.42, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.818359375}]}
{"cur_time": "2024-08-26 04:17:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.406}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 140.932}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.174}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 130.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.193}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 134.57}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 04:18:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 6.75537109375, "powerusage_W": 140.549}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.26971435546875, "powerusage_W": 138.742}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.7349853515625, "powerusage_W": 140.51}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.4830322265625, "powerusage_W": 135.671}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.4869384765625, "powerusage_W": 130.362}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.5670166015625, "powerusage_W": 135.635}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.0142822265625, "powerusage_W": 134.022}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 4.91015625}]}
{"cur_time": "2024-08-26 04:19:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.49560546875, "powerusage_W": 142.845}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.36737060546875, "powerusage_W": 341.015}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 142.725}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.8463134765625, "powerusage_W": 333.995}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.7662353515625, "powerusage_W": 333.442}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.8463134765625, "powerusage_W": 341.998}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.3775634765625, "powerusage_W": 339.185}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3535793, "create_time": 1724645802.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.2734375}]}
{"cur_time": "2024-08-26 04:20:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.178}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.077}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 141.639}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.499}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.259}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.751}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 134.636}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:21:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 11.86474609375, "powerusage_W": 140.539}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.36932373046875, "powerusage_W": 138.102}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.6666259765625, "powerusage_W": 140.928}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7291259765625, "powerusage_W": 136.087}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.5552978515625, "powerusage_W": 130.159}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7291259765625, "powerusage_W": 134.919}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.2916259765625, "powerusage_W": 135.744}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3538113, "create_time": 1724646019.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.1875}]}
{"cur_time": "2024-08-26 04:22:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.52685546875, "powerusage_W": 347.486}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.20916748046875, "powerusage_W": 346.516}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3970947265625, "powerusage_W": 343.166}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3912353515625, "powerusage_W": 339.91}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3111572265625, "powerusage_W": 329.189}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 47.3931884765625, "powerusage_W": 335.663}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 46.9224853515625, "powerusage_W": 333.199}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3538113, "create_time": 1724646019.82, "name": "pt_main_thread", "user": "user", "used_mem_GB": 18.818359375}]}
{"cur_time": "2024-08-26 04:23:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.94677734375, "powerusage_W": 145.015}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.936}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.855}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.929}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.015}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.763}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.027}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:24:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28076171875, "powerusage_W": 351.163}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.524}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.42}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.835}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.073}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.184}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.755}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:25:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28271484375, "powerusage_W": 346.615}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.66}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.723}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.2}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.329}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.484}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.034}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:26:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 17.28271484375, "powerusage_W": 353.821}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.22}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 142.139}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.444}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.078}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.113}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.647}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:27:18", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.73931884765625, "powerusage_W": 309.121}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.485}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5260009765625, "powerusage_W": 250.536}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.3013916015625, "powerusage_W": 137.83}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.264}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8931884765625, "powerusage_W": 136.795}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.031}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:28:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.73541259765625, "powerusage_W": 302.962}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 281.768}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 328.171}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3072509765625, "powerusage_W": 309.734}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 272.093}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 259.733}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.077}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:29:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.21392822265625, "powerusage_W": 304.392}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 278.596}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 313.838}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 312.159}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 277.044}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 248.308}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.164}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:30:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.21392822265625, "powerusage_W": 300.442}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 287.852}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 284.01}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 304.484}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 272.111}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4010009765625, "powerusage_W": 256.086}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.777}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:31:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.69439697265625, "powerusage_W": 304.233}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 285.531}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 287.784}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 313.126}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.573}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.435}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.643}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:32:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.71197509765625, "powerusage_W": 301.677}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 285.758}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 340.98}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 303.952}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.402}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 260.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.513}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:33:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.44439697265625, "powerusage_W": 202.796}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 284.235}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 317.666}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 299.932}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.785}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 254.694}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.296}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:34:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09478759765625, "powerusage_W": 302.032}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 289.987}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 321.66}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 308.381}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.557}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.443}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:35:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.344}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 290.232}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 314.5}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 304.705}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.016}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 246.105}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.446}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:36:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.156}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 284.763}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 306.186}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5416259765625, "powerusage_W": 320.057}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.883}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 252.547}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.477}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:37:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 302.467}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 288.747}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 311.333}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 311.475}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.804}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 255.972}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.067}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:38:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 315.185}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 294.833}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 336.552}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 257.864}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.177}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:39:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 301.547}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 283.939}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 310.811}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 318.432}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.556}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.043}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.426}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:40:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.12799072265625, "powerusage_W": 330.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 291.268}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 301.438}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 297.474}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 284.742}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 264.848}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.815}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:41:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.816}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.087}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 300.562}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.865}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.043}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.918}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.326}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:42:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 298.184}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.347}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 282.174}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 319.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.061}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 258.145}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.145}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:43:19", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 316.107}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.176}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 323.187}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 324.684}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.108}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 250.949}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.833}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:44:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 298.994}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 280.52}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5748291015625, "powerusage_W": 311.194}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 308.844}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 250.865}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.807}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:45:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.21392822265625, "powerusage_W": 296.592}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 283.443}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 295.926}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 296.348}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.179}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 271.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.644}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:46:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 25.22174072265625, "powerusage_W": 202.893}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 289.021}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 287.812}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 304.927}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 272.911}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 245.667}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.666}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:47:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09478759765625, "powerusage_W": 299.412}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.033}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 285.618}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 297.281}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.387}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.333}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.551}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:48:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 296.588}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 275.752}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 322.029}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 305.368}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.571}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 254.08}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.59}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:49:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 309.065}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 281.992}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 308.2}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 313.519}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 276.568}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 249.515}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.473}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:50:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 319.809}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 282.917}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 287.486}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 310.894}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 274.215}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 248.388}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.552}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:51:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 323.631}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.35174560546875, "powerusage_W": 288.612}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 281.007}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 322.148}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 280.361}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 270.356}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.92}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:52:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.09674072265625, "powerusage_W": 329.834}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.372}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 315.794}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 305.481}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 277.005}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 261.191}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.035}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:53:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.12799072265625, "powerusage_W": 318.392}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.114}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 313.704}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 313.394}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.31}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 263.152}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.31}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:54:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 301.105}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.523}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 286.549}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 307.014}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.816}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 260.232}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.473}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:55:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.071}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.597}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 322.809}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 306.806}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 275.25}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 256.787}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.929}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:56:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 299.444}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.333}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 310.849}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 320.875}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 281.792}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 259.856}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.496}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:57:20", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.19635009765625, "powerusage_W": 303.264}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.926}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 294.556}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.6646728515625, "powerusage_W": 301.938}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 273.676}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.5592041015625, "powerusage_W": 259.74}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.058}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:58:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.71783447265625, "powerusage_W": 295.375}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.647}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 280.971}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.52}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 278.487}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 134.997}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.079}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 04:59:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.53033447265625, "powerusage_W": 205.6}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 139.634}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.35369873046875, "powerusage_W": 336.222}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 137.788}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.81854248046875, "powerusage_W": 278.001}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 134.853}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.034}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:00:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.16119384765625, "powerusage_W": 198.94}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.38885498046875, "powerusage_W": 139.249}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.81658935546875, "powerusage_W": 314.017}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 45.1627197265625, "powerusage_W": 137.358}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 75.28143310546875, "powerusage_W": 273.34}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.5709228515625, "powerusage_W": 134.571}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.1021728515625, "powerusage_W": 135.732}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.998046875}]}
{"cur_time": "2024-08-26 05:01:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 293.373}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 343.638}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.01385498046875, "powerusage_W": 331.069}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 329.377}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.47674560546875, "powerusage_W": 313.21}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 320.918}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3267822265625, "powerusage_W": 333.035}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.22265625}]}
{"cur_time": "2024-08-26 05:02:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.30377197265625, "powerusage_W": 289.8}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 333.871}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.11151123046875, "powerusage_W": 343.32}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8150634765625, "powerusage_W": 320.624}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.47869873046875, "powerusage_W": 312.171}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 311.694}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 321.877}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3561889, "create_time": 1724648332.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:03:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 222.564}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 140.786}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.09393310546875, "powerusage_W": 296.307}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 138.052}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.56268310546875, "powerusage_W": 302.971}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 135.81}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6138916015625, "powerusage_W": 136.378}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.509765625}]}
{"cur_time": "2024-08-26 05:04:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.53033447265625, "powerusage_W": 204.315}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.318}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.35369873046875, "powerusage_W": 280.658}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 137.431}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 63.82244873046875, "powerusage_W": 273.621}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 135.653}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.039}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:05:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.65338134765625, "powerusage_W": 201.312}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.58807373046875, "powerusage_W": 140.373}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.93182373046875, "powerusage_W": 325.864}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.6549072265625, "powerusage_W": 137.668}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 73.40057373046875, "powerusage_W": 274.442}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.6861572265625, "powerusage_W": 135.711}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.2174072265625, "powerusage_W": 136.142}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 13.11328125}]}
{"cur_time": "2024-08-26 05:06:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 290.146}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 321.54}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.01385498046875, "powerusage_W": 320.879}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 320.485}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.48065185546875, "powerusage_W": 310.118}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 311.612}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3267822265625, "powerusage_W": 314.87}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.22265625}]}
{"cur_time": "2024-08-26 05:07:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.22760009765625, "powerusage_W": 305.696}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 141.492}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.03729248046875, "powerusage_W": 334.979}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 138.724}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 83.48260498046875, "powerusage_W": 261.276}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 136.618}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 137.572}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3564931, "create_time": 1724648585.26, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:08:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 219.345}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.195}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 329.25}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.3519287109375, "powerusage_W": 137.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 275.95}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.552}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.508}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:09:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 27.84674072265625, "powerusage_W": 198.517}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.86541748046875, "powerusage_W": 139.859}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 67.16815185546875, "powerusage_W": 291.335}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.0474853515625, "powerusage_W": 137.596}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.76776123046875, "powerusage_W": 269.052}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.0474853515625, "powerusage_W": 135.562}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.6627197265625, "powerusage_W": 135.315}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.55859375}]}
{"cur_time": "2024-08-26 05:10:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19049072265625, "powerusage_W": 296.992}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.58612060546875, "powerusage_W": 140.517}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.90643310546875, "powerusage_W": 339.783}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8111572265625, "powerusage_W": 138.153}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.6881103515625, "powerusage_W": 132.165}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.7681884765625, "powerusage_W": 136.288}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.2974853515625, "powerusage_W": 136.687}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.193359375}]}
{"cur_time": "2024-08-26 05:11:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.22760009765625, "powerusage_W": 296.481}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.65838623046875, "powerusage_W": 146.492}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 80.03729248046875, "powerusage_W": 348.138}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8131103515625, "powerusage_W": 142.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.6900634765625, "powerusage_W": 135.137}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.8463134765625, "powerusage_W": 138.476}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.3814697265625, "powerusage_W": 141.262}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3568478, "create_time": 1724648893.16, "name": "pt_main_thread", "user": "user", "used_mem_GB": 23.27734375}]}
{"cur_time": "2024-08-26 05:12:21", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 16.80224609375, "powerusage_W": 205.622}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 145.138}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 293.224}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.815}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 134.074}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.218}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.969}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:13:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.57958984375, "powerusage_W": 143.707}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.715}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 341.945}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.705}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.991}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.454}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:14:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.65771484375, "powerusage_W": 352.01}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.774}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 337.113}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.407}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.919}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.781}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.709}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:15:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 345.692}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.773}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 329.886}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.382}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.796}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.625}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.475}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:16:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 335.762}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.361}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 308.23}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.726}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.426}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.142}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.053}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:17:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.84521484375, "powerusage_W": 334.038}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.687}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 305.197}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.105}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.811}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.913}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:18:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 348.407}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.593}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 291.587}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.21}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.712}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.461}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.635}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:19:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 339.992}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.379}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 309.034}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.93}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.481}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.371}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.42}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:20:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 147.13}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.569}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 332.456}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.777}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.367}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.286}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.303}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:21:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.87255859375, "powerusage_W": 142.14}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.456}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.69}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.517}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.344}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.314}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.226}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:22:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.89361572265625, "powerusage_W": 346.515}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.88690185546875, "powerusage_W": 375.662}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 324.634}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.1353759765625, "powerusage_W": 365.74}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.4967041015625, "powerusage_W": 367.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.2427978515625, "powerusage_W": 360.64}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.102}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:23:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.91705322265625, "powerusage_W": 354.116}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 381.866}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 308.342}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 370.12}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 365.19}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.7838134765625, "powerusage_W": 363.207}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.952}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:24:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.96002197265625, "powerusage_W": 369.53}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 381.09}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 299.485}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.753}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 367.08}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0474853515625, "powerusage_W": 360.675}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.521}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:25:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.46978759765625, "powerusage_W": 362.192}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 388.85}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 297.359}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 369.577}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 360.384}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0474853515625, "powerusage_W": 370.529}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.241}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:26:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.46978759765625, "powerusage_W": 357.594}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 385.923}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.906}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 375.82}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 359.126}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 359.493}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.226}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:27:22", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.74713134765625, "powerusage_W": 349.798}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 378.301}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 281.159}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.651}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 363.695}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 377.954}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.188}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:28:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.74713134765625, "powerusage_W": 360.176}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.01385498046875, "powerusage_W": 385.96}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 325.173}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 378.9}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 369.781}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 361.498}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.151}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:29:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 52.77447509765625, "powerusage_W": 360.805}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 382.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 311.623}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 367.214}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 368.797}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 361.639}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.104}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:30:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 346.449}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 387.003}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.216}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 375.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 363.13}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 368.138}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.071}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:31:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 364.147}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.26971435546875, "powerusage_W": 383.851}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 292.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0318603515625, "powerusage_W": 369.372}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.8092041015625, "powerusage_W": 362.68}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 370.508}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.02}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:32:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 370.093}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 148.674}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 289.793}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.2877197265625, "powerusage_W": 384.687}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.7740478515625, "powerusage_W": 366.49}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.0494384765625, "powerusage_W": 365.32}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 139.057}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:33:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 366.65}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.15}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 307.499}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.585}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.508}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.088}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.345}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:34:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 359.879}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.269}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.268}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 140.248}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.996}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.185}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.492}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:35:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.22564697265625, "powerusage_W": 353.193}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 143.374}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 290.79}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.982}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 133.059}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.245}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.652}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:36:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.71783447265625, "powerusage_W": 387.943}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.874}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 284.587}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.758}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.654}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.755}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.269}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:37:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.27252197265625, "powerusage_W": 362.326}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 336.232}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.629}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.281}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.969}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:38:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 71.8502197265625, "powerusage_W": 347.998}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.893}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 321.0}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.273}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.015}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.07}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.73}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:39:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 357.978}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.634}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 306.785}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 139.105}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.811}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.905}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.633}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:40:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 345.959}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.433}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 293.819}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.371}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.607}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.586}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.277}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:41:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 363.674}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.389}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 313.263}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.853}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.403}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.56}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.294}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:42:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.03033447265625, "powerusage_W": 342.667}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.001}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 304.662}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.656}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.323}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.428}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.121}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:43:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.72412109375, "powerusage_W": 341.943}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.158}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 290.737}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.72}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.737}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.784}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.167}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:44:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 22.43463134765625, "powerusage_W": 340.705}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 141.837}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.14471435546875, "powerusage_W": 338.6}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 137.601}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.2713623046875, "powerusage_W": 131.996}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.538}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:45:23", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 347.797}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.493}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 332.982}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.934}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.907}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.985}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:46:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 21.91943359375, "powerusage_W": 336.213}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.487}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 317.586}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.99}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.171}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.285}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.59}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:47:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.21240234375, "powerusage_W": 141.868}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.92596435546875, "powerusage_W": 140.653}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.40447998046875, "powerusage_W": 310.332}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.285}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.0279541015625, "powerusage_W": 131.317}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.1080322265625, "powerusage_W": 136.482}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.6392822265625, "powerusage_W": 136.669}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3591591, "create_time": 1724651217.99, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.53515625}]}
{"cur_time": "2024-08-26 05:48:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.236}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.587}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 294.24}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.304}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.337}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 136.368}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.745}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:49:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 13.70654296875, "powerusage_W": 140.968}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 41.48651123046875, "powerusage_W": 140.683}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 68.89862060546875, "powerusage_W": 281.568}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.4967041015625, "powerusage_W": 136.305}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.5220947265625, "powerusage_W": 131.599}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.8775634765625, "powerusage_W": 136.699}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.1334228515625, "powerusage_W": 136.995}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3592852, "create_time": 1724651301.87, "name": "pt_main_thread", "user": "user", "used_mem_GB": 12.029296875}]}
{"cur_time": "2024-08-26 05:50:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 7.50732421875, "powerusage_W": 140.42}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.01190185546875, "powerusage_W": 139.96}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 62.80487060546875, "powerusage_W": 301.592}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.8131103515625, "powerusage_W": 136.181}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.1138916015625, "powerusage_W": 130.993}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.1939697265625, "powerusage_W": 136.293}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.7252197265625, "powerusage_W": 136.372}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3594123, "create_time": 1724651385.9, "name": "pt_main_thread", "user": "user", "used_mem_GB": 5.62109375}]}
{"cur_time": "2024-08-26 05:51:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 5.15771484375, "powerusage_W": 139.758}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.562}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 279.07}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.615}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.501}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.794}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.884}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:52:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 9.38232421875, "powerusage_W": 139.597}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.05}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 326.272}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.657}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.253}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.54}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.82}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:53:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 143.685}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 141.891}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6275634765625, "powerusage_W": 298.044}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 138.781}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 132.482}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.691}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.956}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 05:54:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 10.51708984375, "powerusage_W": 141.489}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.14666748046875, "powerusage_W": 141.029}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 65.62518310546875, "powerusage_W": 290.889}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.4967041015625, "powerusage_W": 137.325}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.2486572265625, "powerusage_W": 131.757}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.4967041015625, "powerusage_W": 136.924}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 36.8599853515625, "powerusage_W": 137.082}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 8.755859375}]}
{"cur_time": "2024-08-26 05:55:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.68896484375, "powerusage_W": 143.559}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 143.52}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 76.75799560546875, "powerusage_W": 339.56}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5845947265625, "powerusage_W": 139.635}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 132.716}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 137.541}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1158447265625, "powerusage_W": 138.791}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.01171875}]}
{"cur_time": "2024-08-26 05:56:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 143.624}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 144.523}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 348.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 139.493}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 134.199}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 137.737}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 139.251}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3597518, "create_time": 1724651609.93, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 05:57:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 4.62646484375, "powerusage_W": 141.472}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.78143310546875, "powerusage_W": 142.308}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 61.25994873046875, "powerusage_W": 351.051}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 32.9635009765625, "powerusage_W": 138.144}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 30.9888916015625, "powerusage_W": 131.951}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8463134765625, "powerusage_W": 136.224}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6119384765625, "powerusage_W": 137.382}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3599799, "create_time": 1724651831.23, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.5078125}]}
{"cur_time": "2024-08-26 05:58:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 12.57958984375, "powerusage_W": 141.151}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.19940185546875, "powerusage_W": 141.702}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 67.50994873046875, "powerusage_W": 321.731}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.2135009765625, "powerusage_W": 138.208}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.2174072265625, "powerusage_W": 131.378}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 39.3814697265625, "powerusage_W": 136.263}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.7447509765625, "powerusage_W": 136.875}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.640625}]}
{"cur_time": "2024-08-26 05:59:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 313.445}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 320.27}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 335.147}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 309.965}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 293.277}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 309.245}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 308.218}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:00:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 142.483}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 143.063}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 78.88299560546875, "powerusage_W": 355.718}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 138.572}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 132.856}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 136.362}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 138.073}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3600198, "create_time": 1724651844.67, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:01:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 141.214}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.66424560546875, "powerusage_W": 141.626}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.14471435546875, "powerusage_W": 302.053}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 137.957}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.7681884765625, "powerusage_W": 131.548}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.8482666015625, "powerusage_W": 136.452}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.6138916015625, "powerusage_W": 137.342}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 0.509765625}]}
{"cur_time": "2024-08-26 06:02:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.87841796875, "powerusage_W": 140.78}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.34002685546875, "powerusage_W": 141.185}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 72.67205810546875, "powerusage_W": 291.237}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.7740478515625, "powerusage_W": 137.644}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.6939697265625, "powerusage_W": 131.042}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.1451416015625, "powerusage_W": 136.213}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.3052978515625, "powerusage_W": 136.526}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 14.201171875}]}
{"cur_time": "2024-08-26 06:03:24", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 23.69091796875, "powerusage_W": 326.135}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 51.40447998046875, "powerusage_W": 311.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 72.79119873046875, "powerusage_W": 317.325}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 312.02}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5064697265625, "powerusage_W": 306.301}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.5865478515625, "powerusage_W": 136.835}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 50.1177978515625, "powerusage_W": 290.791}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3602784, "create_time": 1724652072.68, "name": "pt_main_thread", "user": "user", "used_mem_GB": 22.013671875}]}
{"cur_time": "2024-08-26 06:04:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 1.48431396484375, "powerusage_W": 142.689}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 142.899}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 308.788}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.789}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 131.676}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 137.429}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.225}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:05:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.48388671875, "powerusage_W": 236.575}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 140.427}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.5767822265625, "powerusage_W": 320.467}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.592}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.665}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.628}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.975}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:06:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47802734375, "powerusage_W": 346.426}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.692}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 290.416}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.689}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.495}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.591}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.899}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:07:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47802734375, "powerusage_W": 343.48}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.444}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 296.314}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.383}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.482}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.514}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.923}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:08:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47998046875, "powerusage_W": 341.683}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.29}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 56.6256103515625, "powerusage_W": 294.092}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.478}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.34}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.457}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.822}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:09:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 20.47998046875, "powerusage_W": 339.951}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.1490478515625, "powerusage_W": 139.181}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 59.09783935546875, "powerusage_W": 289.789}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.266}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.25299072265625, "powerusage_W": 130.173}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.33306884765625, "powerusage_W": 135.302}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.751}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:10:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.63970947265625, "powerusage_W": 295.26}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.41619873046875, "powerusage_W": 280.204}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 387.697}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 43.9420166015625, "powerusage_W": 328.492}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 278.847}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.0299072265625, "powerusage_W": 247.705}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.443}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:11:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 33.63970947265625, "powerusage_W": 300.721}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 281.292}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 343.606}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.0494384765625, "powerusage_W": 318.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0260009765625, "powerusage_W": 281.003}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.0299072265625, "powerusage_W": 249.081}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.427}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:12:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04400634765625, "powerusage_W": 298.248}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 289.231}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.77557373046875, "powerusage_W": 328.141}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2603759765625, "powerusage_W": 312.913}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 282.709}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 255.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.418}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:13:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.04833984375, "powerusage_W": 204.057}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 268.249}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 426.005}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2720947265625, "powerusage_W": 307.349}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 277.829}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 252.688}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.596}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:14:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.55963134765625, "powerusage_W": 295.794}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.141}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 346.662}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.2838134765625, "powerusage_W": 310.742}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0416259765625, "powerusage_W": 276.991}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.1412353515625, "powerusage_W": 248.042}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.635}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:15:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04205322265625, "powerusage_W": 309.223}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.119}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 406.556}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3970947265625, "powerusage_W": 304.214}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.457}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.2545166015625, "powerusage_W": 248.764}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.551}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:16:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.04205322265625, "powerusage_W": 290.733}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 278.664}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 332.399}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.3970947265625, "powerusage_W": 310.685}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 274.854}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.265}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.873}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:17:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45416259765625, "powerusage_W": 314.019}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 291.727}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 353.897}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 328.048}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.313}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 254.51}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 138.61}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:18:25", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 296.616}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 289.929}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 347.718}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 323.908}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 276.58}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 250.95}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.471}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:19:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 293.781}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 292.25}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 342.762}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 323.265}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 274.732}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.128}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.51}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:20:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 315.16}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 290.287}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 336.096}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 319.765}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 273.555}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 264.457}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.656}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:21:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 316.788}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 291.824}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 331.798}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 319.907}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 279.855}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 248.352}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.323}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:22:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 298.644}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 282.371}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 398.965}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 316.996}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 272.595}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 248.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 137.107}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:23:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.5982666015625, "powerusage_W": 315.832}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 54.068115234375, "powerusage_W": 284.261}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 89.450927734375, "powerusage_W": 420.678}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15057373046875, "powerusage_W": 315.424}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 64.67791748046875, "powerusage_W": 277.756}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.00408935546875, "powerusage_W": 246.631}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8287353515625, "powerusage_W": 136.97}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3616690, "create_time": 1724653383.09, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.724609375}]}
{"cur_time": "2024-08-26 06:24:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 37.5982666015625, "powerusage_W": 298.57}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 54.068115234375, "powerusage_W": 288.962}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 89.450927734375, "powerusage_W": 344.163}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.15057373046875, "powerusage_W": 311.452}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 64.67791748046875, "powerusage_W": 291.469}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.00408935546875, "powerusage_W": 247.36}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 31.8287353515625, "powerusage_W": 136.57}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3617351, "create_time": 1724653450.15, "name": "pt_main_thread", "user": "user", "used_mem_GB": 3.724609375}]}
{"cur_time": "2024-08-26 06:25:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4713134765625, "powerusage_W": 285.335}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.909912109375, "powerusage_W": 291.257}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.292724609375, "powerusage_W": 336.679}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.99237060546875, "powerusage_W": 319.649}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.51971435546875, "powerusage_W": 277.728}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.84588623046875, "powerusage_W": 248.545}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3853759765625, "powerusage_W": 136.633}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3617351, "create_time": 1724653450.15, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.279296875}]}
{"cur_time": "2024-08-26 06:26:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.46044921875, "powerusage_W": 223.24}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 282.986}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 408.706}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 321.258}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 276.188}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 247.182}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.409}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:27:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 29.83502197265625, "powerusage_W": 285.327}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 279.326}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 402.052}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 318.29}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0455322265625, "powerusage_W": 277.214}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.3717041015625, "powerusage_W": 252.257}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.171}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:28:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 40.6138916015625, "powerusage_W": 291.112}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.611083984375, "powerusage_W": 279.14}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 93.993896484375, "powerusage_W": 391.64}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.69354248046875, "powerusage_W": 311.88}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.22088623046875, "powerusage_W": 272.858}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.54705810546875, "powerusage_W": 249.828}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 35.4693603515625, "powerusage_W": 136.259}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 7.36328125}]}
{"cur_time": "2024-08-26 06:29:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4263916015625, "powerusage_W": 296.042}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.890380859375, "powerusage_W": 281.25}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.273193359375, "powerusage_W": 328.738}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.97283935546875, "powerusage_W": 311.16}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.50018310546875, "powerusage_W": 282.81}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.82635498046875, "powerusage_W": 248.366}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3736572265625, "powerusage_W": 136.001}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.267578125}]}
{"cur_time": "2024-08-26 06:30:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 42.4263916015625, "powerusage_W": 292.394}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 58.890380859375, "powerusage_W": 286.352}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 94.273193359375, "powerusage_W": 346.719}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 53.97283935546875, "powerusage_W": 317.404}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 69.50213623046875, "powerusage_W": 276.582}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 57.95135498046875, "powerusage_W": 258.118}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 38.3736572265625, "powerusage_W": 135.99}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}, {"pid": 3619916, "create_time": 1724653678.52, "name": "pt_main_thread", "user": "user", "used_mem_GB": 10.267578125}]}
{"cur_time": "2024-08-26 06:31:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45416259765625, "powerusage_W": 297.031}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 285.718}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 346.254}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 310.428}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 275.238}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 248.741}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.944}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:32:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 323.825}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 278.639}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 409.21}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 312.648}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 273.979}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 255.409}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 136.002}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:33:26", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 295.191}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 279.869}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 353.471}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 308.75}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.654}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 245.291}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.788}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:34:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.45611572265625, "powerusage_W": 297.6}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.32}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 340.845}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 308.419}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 279.155}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 246.301}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.788}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:35:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 291.868}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 287.858}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 329.103}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 320.26}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.673}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 248.512}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.524}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:36:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 290.544}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 284.997}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 336.716}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 313.983}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.421}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 261.42}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.544}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:37:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 296.35}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 283.42}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 343.146}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 310.729}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.024}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 245.88}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.52}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:38:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 290.45}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 281.809}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 345.674}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 311.361}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 272.891}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 253.455}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.425}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:39:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 34.88189697265625, "powerusage_W": 301.222}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 280.329}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 342.566}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 309.825}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 274.806}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 261.929}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.591}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:40:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.88623046875, "powerusage_W": 237.753}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 276.164}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 344.464}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 314.211}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 259.566}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 239.195}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.403}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}
{"cur_time": "2024-08-26 06:41:27", "all_gpu_status": [{"device": 0, "total_mem_GB": 95.5771484375, "used_mem_GB": 15.88623046875, "powerusage_W": 198.72}, {"device": 1, "total_mem_GB": 95.5771484375, "used_mem_GB": 49.43768310546875, "powerusage_W": 285.704}, {"device": 2, "total_mem_GB": 95.5771484375, "used_mem_GB": 84.82049560546875, "powerusage_W": 394.92}, {"device": 3, "total_mem_GB": 95.5771484375, "used_mem_GB": 44.5181884765625, "powerusage_W": 311.776}, {"device": 4, "total_mem_GB": 95.5771484375, "used_mem_GB": 60.0474853515625, "powerusage_W": 268.271}, {"device": 5, "total_mem_GB": 95.5771484375, "used_mem_GB": 48.4967041015625, "powerusage_W": 250.608}, {"device": 6, "total_mem_GB": 95.5771484375, "used_mem_GB": 28.09869384765625, "powerusage_W": 135.342}], "all_processes_status": [{"pid": 2422621, "create_time": 1724584908.08, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 2473948, "create_time": 1724591088.49, "name": "python3.8", "user": "root", "used_mem_GB": 0.626953125}, {"pid": 4114854, "create_time": 1724615522.19, "name": "python3.8", "user": "root", "used_mem_GB": 26.240234375}]}

10
results/inference/Baichuan2-7B/Baichuan2_predict_1/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0027,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 133.1946,
"predict_samples_per_second": 0.375,
"predict_steps_per_second": 0.03
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_1/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_1/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0027,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 133.1946,
"predict_samples_per_second": 0.375,
"predict_steps_per_second": 0.03
}

94
results/inference/Baichuan2-7B/Baichuan2_predict_1_log.txt Normal file
View File

@ -0,0 +1,94 @@
[2024-08-26 05:53:27,241] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:53:29 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:24581
[2024-08-26 05:53:34,784] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:35,697] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:35,730] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:35,746] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:35,900] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:36,080] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:53:36,118] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:53:37 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:37 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:37 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:37 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:39 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:39 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:39 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:39 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:39 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:40 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:40 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:53:40 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:53:40 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:53:53 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:00 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:54:55 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:55 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:54:55
08/26/2024 05:54:55 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:55 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:54:55
08/26/2024 05:54:57 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:57 - INFO - llamafactory.model.loader - all params: 7,000,559,616
08/26/2024 05:54:57 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
predict start time: 2024-08-26 05:54:57
08/26/2024 05:54:57 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:54:57
08/26/2024 05:54:57 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:57 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:57 - INFO - llamafactory.model.loader - all params: 7,000,559,616
08/26/2024 05:54:57 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:54:57
predict start time: 2024-08-26 05:54:57
08/26/2024 05:54:57 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:54:57 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:54:57
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
predict end time: 2024-08-26 05:57:10
***** predict metrics *****
predict_bleu-4 = 2.0659
predict_model_preparation_time = 0.0027
predict_rouge-1 = 14.744
predict_rouge-2 = 3.8762
predict_rouge-l = 6.5736
predict_runtime = 0:02:13.19
predict_samples_per_second = 0.375
predict_steps_per_second = 0.03
08/26/2024 05:57:10 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_1/generated_predictions.jsonl

10
results/inference/Baichuan2-7B/Baichuan2_predict_1_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0024,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 474.3243,
"predict_samples_per_second": 0.105,
"predict_steps_per_second": 0.053
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_1_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_1_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0024,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 474.3243,
"predict_samples_per_second": 0.105,
"predict_steps_per_second": 0.053
}

5
results/inference/Baichuan2-7B/Baichuan2_predict_1_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:01:01", "remaining_time": "0:04:07"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:02:19", "remaining_time": "0:03:28"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:03:36", "remaining_time": "0:02:24"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:05:46", "remaining_time": "0:01:26"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:07:34", "remaining_time": "0:00:00"}

25
results/inference/Baichuan2-7B/Baichuan2_predict_1_single_log.txt Normal file
View File

@ -0,0 +1,25 @@
[2024-08-26 06:04:32,725] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 06:04:35 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 06:04:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:04:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:04:35 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 06:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:05:23 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:05:23 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:05:23
predict end time: 2024-08-26 06:13:17
***** predict metrics *****
predict_bleu-4 = 1.6536
predict_model_preparation_time = 0.0024
predict_rouge-1 = 17.5981
predict_rouge-2 = 5.6356
predict_rouge-l = 6.105
predict_runtime = 0:07:54.32
predict_samples_per_second = 0.105
predict_steps_per_second = 0.053
08/26/2024 06:13:18 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_1_single/generated_predictions.jsonl

10
results/inference/Baichuan2-7B/Baichuan2_predict_2/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 133.9805,
"predict_samples_per_second": 0.373,
"predict_steps_per_second": 0.03
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_2/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_2/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 133.9805,
"predict_samples_per_second": 0.373,
"predict_steps_per_second": 0.03
}

94
results/inference/Baichuan2-7B/Baichuan2_predict_2_log.txt Normal file
View File

@ -0,0 +1,94 @@
[2024-08-26 05:57:20,892] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:57:24 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:23881
[2024-08-26 05:57:30,334] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:30,336] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:31,106] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:31,138] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:31,138] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:31,218] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:57:31,494] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:57:35 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 05:57:35 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:57:49 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:57:55 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:58:47 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:47 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:58:47
08/26/2024 05:58:48 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:48 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:48 - INFO - llamafactory.model.loader - all params: 7,000,559,616
08/26/2024 05:58:48 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:58:48
predict start time: 2024-08-26 05:58:48
08/26/2024 05:58:48 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:48 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:58:48
08/26/2024 05:58:49 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:49 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:58:49
08/26/2024 05:58:49 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:58:49 - INFO - llamafactory.model.loader - all params: 7,000,559,616
08/26/2024 05:58:49 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
predict start time: 2024-08-26 05:58:49
08/26/2024 05:58:49 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 05:58:49
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
predict end time: 2024-08-26 06:01:01
***** predict metrics *****
predict_bleu-4 = 2.0659
predict_model_preparation_time = 0.0026
predict_rouge-1 = 14.744
predict_rouge-2 = 3.8762
predict_rouge-l = 6.5736
predict_runtime = 0:02:13.98
predict_samples_per_second = 0.373
predict_steps_per_second = 0.03
08/26/2024 06:01:01 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_2/generated_predictions.jsonl

10
results/inference/Baichuan2-7B/Baichuan2_predict_2_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0024,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 724.4399,
"predict_samples_per_second": 0.069,
"predict_steps_per_second": 0.035
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_2_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_2_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0024,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 724.4399,
"predict_samples_per_second": 0.069,
"predict_steps_per_second": 0.035
}

5
results/inference/Baichuan2-7B/Baichuan2_predict_2_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:02:01", "remaining_time": "0:08:06"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:04:33", "remaining_time": "0:06:50"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:07:06", "remaining_time": "0:04:44"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:09:39", "remaining_time": "0:02:24"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:11:30", "remaining_time": "0:00:00"}

25
results/inference/Baichuan2-7B/Baichuan2_predict_2_single_log.txt Normal file
View File

@ -0,0 +1,25 @@
[2024-08-26 06:13:23,788] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 06:13:27 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 06:13:27 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:13:27 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:13:27 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 06:13:32 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:14:18 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:14:18 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:14:18
predict end time: 2024-08-26 06:26:23
***** predict metrics *****
predict_bleu-4 = 1.6536
predict_model_preparation_time = 0.0024
predict_rouge-1 = 17.5981
predict_rouge-2 = 5.6356
predict_rouge-l = 6.105
predict_runtime = 0:12:04.43
predict_samples_per_second = 0.069
predict_steps_per_second = 0.035
08/26/2024 06:26:23 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_2_single/generated_predictions.jsonl

10
results/inference/Baichuan2-7B/Baichuan2_predict_3/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 107.5849,
"predict_samples_per_second": 0.465,
"predict_steps_per_second": 0.037
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_3/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_3/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 2.0658964285714285,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 14.744025000000002,
"predict_rouge-2": 3.876194642857143,
"predict_rouge-l": 6.573617857142858,
"predict_runtime": 107.5849,
"predict_samples_per_second": 0.465,
"predict_steps_per_second": 0.037
}

94
results/inference/Baichuan2-7B/Baichuan2_predict_3_log.txt Normal file
View File

@ -0,0 +1,94 @@
[2024-08-26 06:01:10,284] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 06:01:12 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:26507
[2024-08-26 06:01:17,553] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:17,790] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:18,644] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:18,761] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:18,782] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:18,803] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 06:01:18,877] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 06:01:20 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:20 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:20 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:20 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:20 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:20 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:20 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:20 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:20 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:22 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:22 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:22 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:22 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:22 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:22 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:22 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:22 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:22 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:22 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:23 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:23 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:23 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 06:01:23 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:01:23 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 06:01:34 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 06:01:39 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:01:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:02:27 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:27 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:27
08/26/2024 06:02:34 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:34 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:34
08/26/2024 06:02:34 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:34 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:34
08/26/2024 06:02:34 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:34 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:34 - INFO - llamafactory.model.loader - all params: 7,000,559,616
08/26/2024 06:02:34 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:34
predict start time: 2024-08-26 06:02:34
08/26/2024 06:02:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:35 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:35
08/26/2024 06:02:36 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:02:36 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:02:36
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:21
predict end time: 2024-08-26 06:04:22
***** predict metrics *****
predict_bleu-4 = 2.0659
predict_model_preparation_time = 0.0026
predict_rouge-1 = 14.744
predict_rouge-2 = 3.8762
predict_rouge-l = 6.5736
predict_runtime = 0:01:47.58
predict_samples_per_second = 0.465
predict_steps_per_second = 0.037
08/26/2024 06:04:22 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_3/generated_predictions.jsonl

10
results/inference/Baichuan2-7B/Baichuan2_predict_3_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 729.0221,
"predict_samples_per_second": 0.069,
"predict_steps_per_second": 0.034
}

50
results/inference/Baichuan2-7B/Baichuan2_predict_3_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Baichuan2-7B/Baichuan2_predict_3_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.6535760000000002,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 17.598072,
"predict_rouge-2": 5.635632,
"predict_rouge-l": 6.104977999999999,
"predict_runtime": 729.0221,
"predict_samples_per_second": 0.069,
"predict_steps_per_second": 0.034
}

5
results/inference/Baichuan2-7B/Baichuan2_predict_3_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:02:07", "remaining_time": "0:08:29"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:04:42", "remaining_time": "0:07:03"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:07:14", "remaining_time": "0:04:49"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:09:47", "remaining_time": "0:02:26"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:11:35", "remaining_time": "0:00:00"}

25
results/inference/Baichuan2-7B/Baichuan2_predict_3_single_log.txt Normal file
View File

@ -0,0 +1,25 @@
[2024-08-26 06:26:28,612] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 06:26:31 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 06:26:31 - INFO - llamafactory.data.template - Add pad token: </s>
08/26/2024 06:26:31 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 06:26:31 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[195, 31106, 4550, 19463, 7841, 7868, 73, 196]
inputs:
<reserved_102> 保持健康的三个提示。<reserved_103>
08/26/2024 06:26:36 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 06:27:21 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 06:27:21 - INFO - llamafactory.model.loader - all params: 7,000,559,616
predict start time: 2024-08-26 06:27:21
predict end time: 2024-08-26 06:39:30
***** predict metrics *****
predict_bleu-4 = 1.6536
predict_model_preparation_time = 0.0031
predict_rouge-1 = 17.5981
predict_rouge-2 = 5.6356
predict_rouge-l = 6.105
predict_runtime = 0:12:09.02
predict_samples_per_second = 0.069
predict_steps_per_second = 0.034
08/26/2024 06:39:30 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Baichuan2-7B/Baichuan2_predict_3_single/generated_predictions.jsonl

74
results/inference/ChatGLM2-6B/ChatGLM2_predict_1_log.txt Normal file
View File

@ -0,0 +1,74 @@
[2024-08-26 05:46:55,650] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:46:57 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:22088
[2024-08-26 05:47:03,048] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:03,879] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:03,967] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:04,218] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:04,385] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:04,396] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:47:04,483] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:47:06 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:06 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:06 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:07 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:07 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:07 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:08 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:08 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:08 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:08 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:08 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:08 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:08 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:08 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:08 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:08 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:08 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:08 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:08 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:47:08 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:47:08 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:47:22 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:47:28 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:10 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:10 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:48:10
08/26/2024 05:48:12 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:12 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:12 - INFO - llamafactory.model.loader - all params: 6,243,584,000
08/26/2024 05:48:12 - INFO - llamafactory.model.loader - all params: 6,243,584,000
08/26/2024 05:48:12 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:12 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:48:12
predict start time: 2024-08-26 05:48:12
predict start time: 2024-08-26 05:48:12
08/26/2024 05:48:12 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:12 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:48:12
08/26/2024 05:48:12 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:12 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:48:12
08/26/2024 05:48:13 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:48:13 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:48:13

17
results/inference/ChatGLM2-6B/ChatGLM2_predict_1_single_log.txt Normal file
View File

@ -0,0 +1,17 @@
[2024-08-26 05:51:06,762] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:51:09 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:51:09 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:51:09 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:51:14 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:51:48 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:51:48 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:51:48

74
results/inference/ChatGLM2-6B/ChatGLM2_predict_2_log.txt Normal file
View File

@ -0,0 +1,74 @@
[2024-08-26 05:48:19,591] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:48:21 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:26121
[2024-08-26 05:48:26,352] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:27,300] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:27,921] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:27,936] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:27,956] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:28,028] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:48:28,040] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:48:29 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:29 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:29 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:30 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:30 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:30 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:31 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:31 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:31 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:31 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:31 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:31 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:31 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:31 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:31 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:32 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:32 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:32 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:48:32 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:48:32 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:32 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:32 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:48:46 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:48:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:49:33 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:33 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:33
08/26/2024 05:49:34 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:34 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:34
08/26/2024 05:49:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:35 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:35
08/26/2024 05:49:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:35 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:35
08/26/2024 05:49:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:35 - INFO - llamafactory.model.loader - all params: 6,243,584,000
08/26/2024 05:49:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:35 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:35
predict start time: 2024-08-26 05:49:35
08/26/2024 05:49:36 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:49:36 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:49:36

17
results/inference/ChatGLM2-6B/ChatGLM2_predict_2_single_log.txt Normal file
View File

@ -0,0 +1,17 @@
[2024-08-26 05:51:54,247] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:51:57 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:51:57 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:51:57 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:52:03 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:52:35 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:52:35 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:52:35

74
results/inference/ChatGLM2-6B/ChatGLM2_predict_3_log.txt Normal file
View File

@ -0,0 +1,74 @@
[2024-08-26 05:49:43,200] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:49:45 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:22721
[2024-08-26 05:49:49,912] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:51,621] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:51,633] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:51,776] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:51,829] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:51,925] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:49:52,013] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:49:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:52 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:52 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:55 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:55 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:55 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:55 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:49:55 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:55 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:55 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:55 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:56 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:56 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:56 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:56 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:56 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:56 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:56 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:49:56 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:49:56 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:56 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:49:56 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:50:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:16 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:50:58 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:50:58 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:50:58
08/26/2024 05:50:59 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:50:59 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:50:59
08/26/2024 05:50:59 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:50:59 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:50:59
08/26/2024 05:50:59 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:50:59 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:50:59
08/26/2024 05:51:00 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:51:00 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:51:00
08/26/2024 05:51:00 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:51:00 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:51:00
08/26/2024 05:51:01 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:51:01 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:51:01

17
results/inference/ChatGLM2-6B/ChatGLM2_predict_3_single_log.txt Normal file
View File

@ -0,0 +1,17 @@
[2024-08-26 05:52:40,610] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:52:43 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:52:43 - INFO - llamafactory.data.template - Cannot add this chat template to tokenizer.
08/26/2024 05:52:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[64790, 64792, 790, 30951, 517, 30910, 30940, 30996, 13, 13, 54761, 31211, 31983, 35959, 32474, 34128, 31155, 13, 13, 55437, 31211]
inputs:
[Round 0]
问：保持健康的三个提示。
答：
08/26/2024 05:52:49 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:53:21 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:53:21 - INFO - llamafactory.model.loader - all params: 6,243,584,000
predict start time: 2024-08-26 05:53:21

10
results/inference/Llama2-7B/llama2_predict_1/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 66.1198,
"predict_samples_per_second": 0.756,
"predict_steps_per_second": 0.06
}

50
results/inference/Llama2-7B/llama2_predict_1/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_1/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 66.1198,
"predict_samples_per_second": 0.756,
"predict_steps_per_second": 0.06
}

80
results/inference/Llama2-7B/llama2_predict_1_log.txt Normal file
View File

@ -0,0 +1,80 @@
[2024-08-26 04:12:28,146] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:12:30 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:20587
[2024-08-26 04:12:35,477] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,059] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,167] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,201] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,480] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,481] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:12:36,481] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:12:38 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:38 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:39 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:39 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:39 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:39 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:39 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:40 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:40 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:40 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:12:40 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:12:52 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:14:37 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:15:26 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:26 - INFO - llamafactory.model.loader - all params: 6,738,415,616
08/26/2024 04:15:26 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
predict start time: 2024-08-26 04:15:26
08/26/2024 04:15:26 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:15:26
08/26/2024 04:15:28 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:28 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:15:28
08/26/2024 04:15:29 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:29 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:15:29
08/26/2024 04:15:29 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:29 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:29 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:15:29 - INFO - llamafactory.model.loader - all params: 6,738,415,616
08/26/2024 04:15:29 - INFO - llamafactory.model.loader - all params: 6,738,415,616
08/26/2024 04:15:29 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:15:29
predict start time: 2024-08-26 04:15:29
predict start time: 2024-08-26 04:15:29
predict end time: 2024-08-26 04:16:32
predict end time: 2024-08-26 04:16:32
predict end time: 2024-08-26 04:16:32
predict end time: 2024-08-26 04:16:32
predict end time: 2024-08-26 04:16:32
***** predict metrics *****
predict_bleu-4 = 1.4568
predict_model_preparation_time = 0.0028
predict_rouge-1 = 11.0984
predict_rouge-2 = 3.0424
predict_rouge-l = 4.9929
predict_runtime = 0:01:06.11
predict_samples_per_second = 0.756
predict_steps_per_second = 0.06
08/26/2024 04:16:32 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_1/generated_predictions.jsonl
predict end time: 2024-08-26 04:16:32
predict end time: 2024-08-26 04:16:32

10
results/inference/Llama2-7B/llama2_predict_1_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 519.5559,
"predict_samples_per_second": 0.096,
"predict_steps_per_second": 0.048
}

50
results/inference/Llama2-7B/llama2_predict_1_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_1_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 519.5559,
"predict_samples_per_second": 0.096,
"predict_steps_per_second": 0.048
}

5
results/inference/Llama2-7B/llama2_predict_1_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:00:52", "remaining_time": "0:03:29"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:02:05", "remaining_time": "0:03:08"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:03:24", "remaining_time": "0:02:16"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:05:53", "remaining_time": "0:01:28"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:08:21", "remaining_time": "0:00:00"}

23
results/inference/Llama2-7B/llama2_predict_1_single_log.txt Normal file
View File

@ -0,0 +1,23 @@
[2024-08-26 04:22:55,386] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:22:58 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 04:22:59 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:23:05 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:23:48 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:23:48 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:23:48
predict end time: 2024-08-26 04:32:28
***** predict metrics *****
predict_bleu-4 = 1.2952
predict_model_preparation_time = 0.0031
predict_rouge-1 = 12.8381
predict_rouge-2 = 3.2551
predict_rouge-l = 5.1021
predict_runtime = 0:08:39.55
predict_samples_per_second = 0.096
predict_steps_per_second = 0.048
08/26/2024 04:32:28 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_1_single/generated_predictions.jsonl

10
results/inference/Llama2-7B/llama2_predict_2/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0027,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 67.1806,
"predict_samples_per_second": 0.744,
"predict_steps_per_second": 0.06
}

50
results/inference/Llama2-7B/llama2_predict_2/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_2/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0027,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 67.1806,
"predict_samples_per_second": 0.744,
"predict_steps_per_second": 0.06
}

80
results/inference/Llama2-7B/llama2_predict_2_log.txt Normal file
View File

@ -0,0 +1,80 @@
[2024-08-26 04:16:40,536] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:16:42 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:21982
[2024-08-26 04:16:47,183] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,027] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,404] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,504] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,507] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,548] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:16:48,599] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:16:49 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:49 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:50 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:50 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:51 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:16:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:52 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:52 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:52 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:52 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:16:52 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:16:52 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:17:08 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:18:13 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:19:00 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:00 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:00
08/26/2024 04:19:01 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:01 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:01
08/26/2024 04:19:03 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:03 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:03
08/26/2024 04:19:04 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:04 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:04
08/26/2024 04:19:04 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:04 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:04
08/26/2024 04:19:04 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:04 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:04
08/26/2024 04:19:05 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:19:05 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:19:05
predict end time: 2024-08-26 04:20:08
***** predict metrics *****
predict_bleu-4 = 1.4568
predict_model_preparation_time = 0.0027
predict_rouge-1 = 11.0984
predict_rouge-2 = 3.0424
predict_rouge-l = 4.9929
predict_runtime = 0:01:07.18
predict_samples_per_second = 0.744
predict_steps_per_second = 0.06
08/26/2024 04:20:08 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_2/generated_predictions.jsonl
predict end time: 2024-08-26 04:20:08
predict end time: 2024-08-26 04:20:08
predict end time: 2024-08-26 04:20:08
predict end time: 2024-08-26 04:20:08
predict end time: 2024-08-26 04:20:08
predict end time: 2024-08-26 04:20:08

10
results/inference/Llama2-7B/llama2_predict_2_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 730.1303,
"predict_samples_per_second": 0.068,
"predict_steps_per_second": 0.034
}

50
results/inference/Llama2-7B/llama2_predict_2_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_2_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 730.1303,
"predict_samples_per_second": 0.068,
"predict_steps_per_second": 0.034
}

5
results/inference/Llama2-7B/llama2_predict_2_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:01:45", "remaining_time": "0:07:02"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:04:13", "remaining_time": "0:06:20"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:06:41", "remaining_time": "0:04:27"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:09:09", "remaining_time": "0:02:17"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:11:37", "remaining_time": "0:00:00"}

23
results/inference/Llama2-7B/llama2_predict_2_single_log.txt Normal file
View File

@ -0,0 +1,23 @@
[2024-08-26 04:32:32,815] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:32:35 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 04:32:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:32:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:33:26 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:33:26 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:33:26
predict end time: 2024-08-26 04:45:36
***** predict metrics *****
predict_bleu-4 = 1.2952
predict_model_preparation_time = 0.0028
predict_rouge-1 = 12.8381
predict_rouge-2 = 3.2551
predict_rouge-l = 5.1021
predict_runtime = 0:12:10.13
predict_samples_per_second = 0.068
predict_steps_per_second = 0.034
08/26/2024 04:45:36 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_2_single/generated_predictions.jsonl

10
results/inference/Llama2-7B/llama2_predict_3/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 64.4084,
"predict_samples_per_second": 0.776,
"predict_steps_per_second": 0.062
}

50
results/inference/Llama2-7B/llama2_predict_3/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_3/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4568464285714284,
"predict_model_preparation_time": 0.0031,
"predict_rouge-1": 11.098439285714287,
"predict_rouge-2": 3.0424017857142855,
"predict_rouge-l": 4.992862499999999,
"predict_runtime": 64.4084,
"predict_samples_per_second": 0.776,
"predict_steps_per_second": 0.062
}

80
results/inference/Llama2-7B/llama2_predict_3_log.txt Normal file
View File

@ -0,0 +1,80 @@
[2024-08-26 04:20:17,660] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:20:19 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:25799
[2024-08-26 04:20:24,144] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,045] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,128] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,799] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,835] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,857] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:20:25,895] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:20:26 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:26 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:28 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:28 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:28 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:28 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:29 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:29 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:29 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:29 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:29 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:29 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:29 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:29 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:20:29 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:20:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:20:51 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:21:42 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:42 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:42
08/26/2024 04:21:42 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:42 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:42
08/26/2024 04:21:42 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:42 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:42
08/26/2024 04:21:42 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:42 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:42
08/26/2024 04:21:43 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:43 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:43
08/26/2024 04:21:43 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:43 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:43
08/26/2024 04:21:43 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:21:43 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:21:43
predict end time: 2024-08-26 04:22:46
predict end time: 2024-08-26 04:22:46
predict end time: 2024-08-26 04:22:46
predict end time: 2024-08-26 04:22:46
***** predict metrics *****
predict_bleu-4 = 1.4568
predict_model_preparation_time = 0.0031
predict_rouge-1 = 11.0984
predict_rouge-2 = 3.0424
predict_rouge-l = 4.9929
predict_runtime = 0:01:04.40
predict_samples_per_second = 0.776
predict_steps_per_second = 0.062
08/26/2024 04:22:46 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_3/generated_predictions.jsonl
predict end time: 2024-08-26 04:22:46
predict end time: 2024-08-26 04:22:46
predict end time: 2024-08-26 04:22:46

10
results/inference/Llama2-7B/llama2_predict_3_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 730.3668,
"predict_samples_per_second": 0.068,
"predict_steps_per_second": 0.034
}

50
results/inference/Llama2-7B/llama2_predict_3_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Llama2-7B/llama2_predict_3_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.2952020000000002,
"predict_model_preparation_time": 0.0028,
"predict_rouge-1": 12.838101999999997,
"predict_rouge-2": 3.255076,
"predict_rouge-l": 5.102102,
"predict_runtime": 730.3668,
"predict_samples_per_second": 0.068,
"predict_steps_per_second": 0.034
}

5
results/inference/Llama2-7B/llama2_predict_3_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:01:45", "remaining_time": "0:07:02"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:04:13", "remaining_time": "0:06:20"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:06:41", "remaining_time": "0:04:27"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:09:10", "remaining_time": "0:02:17"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:11:38", "remaining_time": "0:00:00"}

23
results/inference/Llama2-7B/llama2_predict_3_single_log.txt Normal file
View File

@ -0,0 +1,23 @@
[2024-08-26 04:45:41,023] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:45:43 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 04:45:44 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[1, 518, 25580, 29962, 29871, 30982, 31695, 31863, 31577, 30210, 30457, 30502, 31302, 30858, 30267, 518, 29914, 25580, 29962]
inputs:
<s> [INST] 保持健康的三个提示。 [/INST]
08/26/2024 04:45:50 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:46:34 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
08/26/2024 04:46:34 - INFO - llamafactory.model.loader - all params: 6,738,415,616
predict start time: 2024-08-26 04:46:34
predict end time: 2024-08-26 04:58:45
***** predict metrics *****
predict_bleu-4 = 1.2952
predict_model_preparation_time = 0.0028
predict_rouge-1 = 12.8381
predict_rouge-2 = 3.2551
predict_rouge-l = 5.1021
predict_runtime = 0:12:10.36
predict_samples_per_second = 0.068
predict_steps_per_second = 0.034
08/26/2024 04:58:45 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Llama2-7B/llama2_predict_3_single/generated_predictions.jsonl

10
results/inference/Qwen-7B/Qwen_predict_1/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 141.8052,
"predict_samples_per_second": 0.353,
"predict_steps_per_second": 0.028
}

50
results/inference/Qwen-7B/Qwen_predict_1/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_1/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 141.8052,
"predict_samples_per_second": 0.353,
"predict_steps_per_second": 0.028
}

99
results/inference/Qwen-7B/Qwen_predict_1_log.txt Normal file
View File

@ -0,0 +1,99 @@
[2024-08-26 04:58:49,729] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:58:51 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:28824
[2024-08-26 04:58:56,873] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:57,261] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:58,340] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:58,685] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:58,713] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:58,749] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 04:58:58,768] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 04:58:59 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:58:59 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:58:59 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:58:59 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:00 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:00 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:01 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:01 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:02 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:02 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:02 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:02 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:02 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:02 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:03 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:03 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:03 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 04:59:03 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 04:59:03 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 04:59:18 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 04:59:32 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 04:59:33 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:00:23 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:23 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:23
08/26/2024 05:00:29 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:29 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:29
08/26/2024 05:00:30 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:30 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:30
08/26/2024 05:00:30 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:30 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:30
08/26/2024 05:00:30 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:30 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:30
08/26/2024 05:00:30 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:30 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:30
08/26/2024 05:00:30 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:00:30 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:00:30
predict end time: 2024-08-26 05:02:51
predict end time: 2024-08-26 05:02:51
***** predict metrics *****
predict_bleu-4 = 1.9165
predict_model_preparation_time = 0.0026
predict_rouge-1 = 13.0505
predict_rouge-2 = 3.4677
predict_rouge-l = 6.2705
predict_runtime = 0:02:21.80
predict_samples_per_second = 0.353
predict_steps_per_second = 0.028
08/26/2024 05:02:51 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_1/generated_predictions.jsonl
predict end time: 2024-08-26 05:02:51
predict end time: 2024-08-26 05:02:51
predict end time: 2024-08-26 05:02:51
predict end time: 2024-08-26 05:02:51
predict end time: 2024-08-26 05:02:51

10
results/inference/Qwen-7B/Qwen_predict_1_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.003,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 410.6018,
"predict_samples_per_second": 0.122,
"predict_steps_per_second": 0.061
}

50
results/inference/Qwen-7B/Qwen_predict_1_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_1_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.003,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 410.6018,
"predict_samples_per_second": 0.122,
"predict_steps_per_second": 0.061
}

5
results/inference/Qwen-7B/Qwen_predict_1_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:01:04", "remaining_time": "0:04:19"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:02:26", "remaining_time": "0:03:39"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:03:47", "remaining_time": "0:02:31"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:05:09", "remaining_time": "0:01:17"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:06:30", "remaining_time": "0:00:00"}

30
results/inference/Qwen-7B/Qwen_predict_1_single_log.txt Normal file
View File

@ -0,0 +1,30 @@
[2024-08-26 05:12:23,952] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:12:26 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:12:26 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:12:26 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:12:26 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 05:12:46 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:13:32 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:13:32 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:13:32
predict end time: 2024-08-26 05:20:23
***** predict metrics *****
predict_bleu-4 = 1.4984
predict_model_preparation_time = 0.003
predict_rouge-1 = 10.9847
predict_rouge-2 = 2.8472
predict_rouge-l = 5.2718
predict_runtime = 0:06:50.60
predict_samples_per_second = 0.122
predict_steps_per_second = 0.061
08/26/2024 05:20:23 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_1_single/generated_predictions.jsonl

10
results/inference/Qwen-7B/Qwen_predict_2/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 144.7073,
"predict_samples_per_second": 0.346,
"predict_steps_per_second": 0.028
}

50
results/inference/Qwen-7B/Qwen_predict_2/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_2/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 144.7073,
"predict_samples_per_second": 0.346,
"predict_steps_per_second": 0.028
}

99
results/inference/Qwen-7B/Qwen_predict_2_log.txt Normal file
View File

@ -0,0 +1,99 @@
[2024-08-26 05:03:02,821] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:03:05 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:23473
[2024-08-26 05:03:11,414] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:11,677] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:12,185] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:12,225] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:12,262] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:12,290] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:03:12,307] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:03:15 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:15 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:15 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:15 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:15 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:15 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:16 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:16 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:16 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:16 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:16 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:16 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:03:16 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:16 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:03:17 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:03:36 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 05:04:40 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:04:41 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:05:36 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:36 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:36
08/26/2024 05:05:38 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:38 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:38
08/26/2024 05:05:38 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:38 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:38
08/26/2024 05:05:38 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:38 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:38
08/26/2024 05:05:39 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:39 - INFO - llamafactory.model.loader - all params: 7,721,324,544
08/26/2024 05:05:39 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:39 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:39
predict start time: 2024-08-26 05:05:39
08/26/2024 05:05:39 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:05:39 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:05:39
predict end time: 2024-08-26 05:08:01
***** predict metrics *****
predict_bleu-4 = 1.9165
predict_model_preparation_time = 0.0026
predict_rouge-1 = 13.0505
predict_rouge-2 = 3.4677
predict_rouge-l = 6.2705
predict_runtime = 0:02:24.70
predict_samples_per_second = 0.346
predict_steps_per_second = 0.028
08/26/2024 05:08:01 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_2/generated_predictions.jsonl
predict end time: 2024-08-26 05:08:01
predict end time: 2024-08-26 05:08:01
predict end time: 2024-08-26 05:08:01
predict end time: 2024-08-26 05:08:01
predict end time: 2024-08-26 05:08:01
predict end time: 2024-08-26 05:08:01

10
results/inference/Qwen-7B/Qwen_predict_2_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.0034,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 858.812,
"predict_samples_per_second": 0.058,
"predict_steps_per_second": 0.029
}

50
results/inference/Qwen-7B/Qwen_predict_2_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_2_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.0034,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 858.812,
"predict_samples_per_second": 0.058,
"predict_steps_per_second": 0.029
}

5
results/inference/Qwen-7B/Qwen_predict_2_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:02:16", "remaining_time": "0:09:07"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:05:07", "remaining_time": "0:07:41"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:07:59", "remaining_time": "0:05:19"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:10:50", "remaining_time": "0:02:42"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:13:42", "remaining_time": "0:00:00"}

30
results/inference/Qwen-7B/Qwen_predict_2_single_log.txt Normal file
View File

@ -0,0 +1,30 @@
[2024-08-26 05:20:27,259] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:20:29 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:20:30 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:20:30 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:20:30 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 05:20:48 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:21:32 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:21:32 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:21:32
predict end time: 2024-08-26 05:35:51
***** predict metrics *****
predict_bleu-4 = 1.4984
predict_model_preparation_time = 0.0034
predict_rouge-1 = 10.9847
predict_rouge-2 = 2.8472
predict_rouge-l = 5.2718
predict_runtime = 0:14:18.81
predict_samples_per_second = 0.058
predict_steps_per_second = 0.029
08/26/2024 05:35:51 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_2_single/generated_predictions.jsonl

10
results/inference/Qwen-7B/Qwen_predict_3/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 144.1339,
"predict_samples_per_second": 0.347,
"predict_steps_per_second": 0.028
}

50
results/inference/Qwen-7B/Qwen_predict_3/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_3/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.9164732142857144,
"predict_model_preparation_time": 0.0026,
"predict_rouge-1": 13.05053392857143,
"predict_rouge-2": 3.4676642857142856,
"predict_rouge-l": 6.270455357142857,
"predict_runtime": 144.1339,
"predict_samples_per_second": 0.347,
"predict_steps_per_second": 0.028
}

99
results/inference/Qwen-7B/Qwen_predict_3_log.txt Normal file
View File

@ -0,0 +1,99 @@
[2024-08-26 05:08:10,371] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:08:13 - INFO - llamafactory.cli - Initializing distributed tasks at: 127.0.0.1:21535
[2024-08-26 05:08:18,515] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,070] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,875] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,969] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,970] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,977] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-08-26 05:08:19,989] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:08:21 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:21 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:21 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:21 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:21 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:21 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:21 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:22 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:22 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:24 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:24 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:24 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:24 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:24 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:24 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:24 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:24 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:24 - WARNING - llamafactory.hparams.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
08/26/2024 05:08:24 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: None
08/26/2024 05:08:24 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:24 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:24 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:24 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:08:25 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
08/26/2024 05:08:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 05:08:53 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:08:54 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:09:43 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:43 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:43
08/26/2024 05:09:49 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:49 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:49
08/26/2024 05:09:50 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:50 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:50
08/26/2024 05:09:50 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:50 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:50
08/26/2024 05:09:51 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:51 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:51
08/26/2024 05:09:51 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:51 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:51
08/26/2024 05:09:51 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:09:51 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:09:51
predict end time: 2024-08-26 05:12:13
predict end time: 2024-08-26 05:12:13
predict end time: 2024-08-26 05:12:13
predict end time: 2024-08-26 05:12:13
***** predict metrics *****
predict_bleu-4 = 1.9165
predict_model_preparation_time = 0.0026
predict_rouge-1 = 13.0505
predict_rouge-2 = 3.4677
predict_rouge-l = 6.2705
predict_runtime = 0:02:24.13
predict_samples_per_second = 0.347
predict_steps_per_second = 0.028
08/26/2024 05:12:13 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_3/generated_predictions.jsonl
predict end time: 2024-08-26 05:12:13
predict end time: 2024-08-26 05:12:13
predict end time: 2024-08-26 05:12:13

10
results/inference/Qwen-7B/Qwen_predict_3_single/all_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.0023,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 588.5428,
"predict_samples_per_second": 0.085,
"predict_steps_per_second": 0.042
}

50
results/inference/Qwen-7B/Qwen_predict_3_single/generated_predictions.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

10
results/inference/Qwen-7B/Qwen_predict_3_single/predict_results.json Normal file
View File

@ -0,0 +1,10 @@
{
"predict_bleu-4": 1.4983739999999997,
"predict_model_preparation_time": 0.0023,
"predict_rouge-1": 10.984715999999999,
"predict_rouge-2": 2.84725,
"predict_rouge-l": 5.271764000000001,
"predict_runtime": 588.5428,
"predict_samples_per_second": 0.085,
"predict_steps_per_second": 0.042
}

5
results/inference/Qwen-7B/Qwen_predict_3_single/trainer_log.jsonl Normal file
View File

@ -0,0 +1,5 @@
{"current_steps": 5, "total_steps": 25, "percentage": 20.0, "elapsed_time": "0:02:16", "remaining_time": "0:09:06"}
{"current_steps": 10, "total_steps": 25, "percentage": 40.0, "elapsed_time": "0:05:04", "remaining_time": "0:07:36"}
{"current_steps": 15, "total_steps": 25, "percentage": 60.0, "elapsed_time": "0:06:25", "remaining_time": "0:04:16"}
{"current_steps": 20, "total_steps": 25, "percentage": 80.0, "elapsed_time": "0:07:50", "remaining_time": "0:01:57"}
{"current_steps": 25, "total_steps": 25, "percentage": 100.0, "elapsed_time": "0:09:10", "remaining_time": "0:00:00"}

30
results/inference/Qwen-7B/Qwen_predict_3_single_log.txt Normal file
View File

@ -0,0 +1,30 @@
[2024-08-26 05:35:55,483] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
08/26/2024 05:35:57 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: None
08/26/2024 05:35:58 - INFO - llamafactory.data.template - Add eos token: <|im_end|>
08/26/2024 05:35:58 - INFO - llamafactory.data.template - Add pad token: <|im_end|>
08/26/2024 05:35:58 - INFO - llamafactory.data.loader - Loading dataset llamafactory/alpaca_gpt4_zh...
eval example:
input_ids:
[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 100662, 108136, 101124, 45139, 1773, 151645, 198, 151644, 77091, 198]
inputs:
<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
保持健康的三个提示。<|im_end|>
<|im_start|>assistant
08/26/2024 05:36:17 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.
08/26/2024 05:37:03 - INFO - llamafactory.model.model_utils.attention - Using vanilla attention implementation.
08/26/2024 05:37:03 - INFO - llamafactory.model.loader - all params: 7,721,324,544
predict start time: 2024-08-26 05:37:03
predict end time: 2024-08-26 05:46:51
***** predict metrics *****
predict_bleu-4 = 1.4984
predict_model_preparation_time = 0.0023
predict_rouge-1 = 10.9847
predict_rouge-2 = 2.8472
predict_rouge-l = 5.2718
predict_runtime = 0:09:48.54
predict_samples_per_second = 0.085
predict_steps_per_second = 0.042
08/26/2024 05:46:51 - INFO - llamafactory.train.sft.trainer - Saving prediction results to ./results/inference/Qwen-7B/Qwen_predict_3_single/generated_predictions.jsonl