LLaMA-Factory-310P3/npu_status.sh

#!/bin/bash

output_dir=$1
sleep_time=$2
print_to_screen=$3

output_file_path="${output_dir}/gpu_status_$(date +"%Y%m%d%H%M%S").json"

device_cnt=$(npu-smi info | grep '310P3' | wc -l)
chip_id_arr_from_info=(0 1 0 1 0 1 0 1)
npu_id_arr_from_info=(2 2 3 3 5 5 6 6)
npu_id_arr=(2 3 5 6)


get_power_dissipation() {
    local npu_id="$1"
    power_dissipation=$(npu-smi info -t power -i ${npu_id} | grep 'Power Dissipation(W)' | awk '{print $4}')
    echo "${power_dissipation}"
}

get_mem_usage() {
    local chip="$1"
    local device="$2"
    mem_usage=$(npu-smi info | grep "${chip}       ${device}" | awk '{print $8}')
    echo "${mem_usage}"
}


loop_cnt=0

while true; do
    device_mem_usage=""
    for ((i=0; i<device_cnt; i++)); do
        mem_usage=$(get_mem_usage ${chip_id_arr_from_info[i]} ${i})
        device_mem_usage="${device_mem_usage}{\"npu_id\": ${npu_id_arr_from_info[i]}, \"chip_id\": ${chip_id_arr_from_info[i]}, \"device_id\": ${i}, \"mem_usage\": ${mem_usage}}, "
    done
    device_mem_usage="${device_mem_usage%??}"

    npu_power_dissipation=""
    for npu_id in "${npu_id_arr[@]}"; do
        power_dissipation=$(get_power_dissipation ${npu_id})
        npu_power_dissipation="${npu_power_dissipation}{\"npu_id\": ${npu_id}, \"power_dissipation\": ${power_dissipation}}, "
    done
    npu_power_dissipation="${npu_power_dissipation%??}"

    cur_time=$(date +"%Y-%m-%d %H:%M:%S")

    json="{\"cur_time\": \"${cur_time}\", \"npu_power_dissipation\": [${npu_power_dissipation}], \"device_mem_usage\": [${device_mem_usage}]}"
    echo "$json" >> output_file_path
    sleep "${sleep_time}"

    if [ "$print_to_screen" -eq "1" ]; then
        echo "$json"
    fi

    loop_cnt+=1
    if [ "$loop_cnt" -eq "1500" ]; then
        exit 0
    fi

done