diff --git a/src/gpu_status.py b/src/gpu_status.py index 76154695..5b34e264 100644 --- a/src/gpu_status.py +++ b/src/gpu_status.py @@ -1,6 +1,8 @@ import json import pynvml import time +import psutil + def main(): UNIT = 1024 * 1024 * 1024 @@ -22,13 +24,30 @@ def main(): powerusage_W = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000 )) + all_processes_status = [] + pidAllInfo = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + for pidInfo in pidAllInfo: + pidUser = psutil.Process(pidInfo.pid).username() + pidCreateTime = psutil.Process(pidInfo.pid).create_time() + pidName = psutil.Process(pidInfo.pid).name() + all_gpu_status.append(dict( + pid = pidInfo.pid, + create_time = pidCreateTime, + name = pidName, + user = pidUser, + used_mem_GB = pidInfo.usedGpuMemory/UNIT + )) + logs = dict( cur_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), - all_gpu_status = all_gpu_status + all_gpu_status = all_gpu_status, + all_processes_status = all_processes_status ) + with open("./results/gpu_status_0821.json", "a", encoding="utf-8") as f: f.write(json.dumps(logs) + "\n") print(logs) + time.sleep(60) pynvml.nvmlShutdown()