147 lines
6.0 KiB
Python
147 lines
6.0 KiB
Python
import pynvml
|
|
import time
|
|
from .utils import connect_print
|
|
|
|
def get_gpu_info():
|
|
"""Retrieves detailed information about installed NVIDIA GPUs"""
|
|
try:
|
|
# Initialize NVML
|
|
pynvml.nvmlInit()
|
|
|
|
# Number of GPUs
|
|
device_count = pynvml.nvmlDeviceGetCount()
|
|
gpu_infos = []
|
|
|
|
for i in range(device_count):
|
|
handle = pynvml.nvmlDeviceGetHandleByIndex(i)
|
|
|
|
# GPU name
|
|
name = pynvml.nvmlDeviceGetName(handle)
|
|
|
|
# GPU utilization
|
|
utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
|
|
gpu_util = utilization.gpu
|
|
|
|
# Memory
|
|
memory = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
memory_total = memory.total / 1024**2 # In MB
|
|
memory_used = memory.used / 1024**2 # In MB
|
|
memory_percent = (memory.used / memory.total) * 100
|
|
|
|
# Temperature
|
|
temperature = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
|
|
|
|
# Power
|
|
try:
|
|
power_usage = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0 # In watts
|
|
power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000.0 # In watts
|
|
except pynvml.NVMLError:
|
|
power_usage = 0
|
|
power_limit = 0
|
|
|
|
# Fans
|
|
try:
|
|
fan_speed = pynvml.nvmlDeviceGetFanSpeed(handle)
|
|
except pynvml.NVMLError:
|
|
fan_speed = 0
|
|
|
|
# Clock frequencies (GPU and memory)
|
|
try:
|
|
clock_info = {
|
|
"graphics_clock": pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS),
|
|
"memory_clock": pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_MEM),
|
|
"sm_clock": pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_SM)
|
|
}
|
|
except pynvml.NVMLError:
|
|
clock_info = {
|
|
"graphics_clock": 0,
|
|
"memory_clock": 0,
|
|
"sm_clock": 0
|
|
}
|
|
|
|
# PCIe utilization
|
|
try:
|
|
pcie_throughput_tx = pynvml.nvmlDeviceGetPcieThroughput(handle, pynvml.NVML_PCIE_UTIL_TX_BYTES)
|
|
pcie_throughput_rx = pynvml.nvmlDeviceGetPcieThroughput(handle, pynvml.NVML_PCIE_UTIL_RX_BYTES)
|
|
pcie_generation = pynvml.nvmlDeviceGetMaxPcieLinkGeneration(handle)
|
|
pcie_width_max = pynvml.nvmlDeviceGetMaxPcieLinkWidth(handle)
|
|
pcie_width_current = pynvml.nvmlDeviceGetCurrPcieLinkWidth(handle)
|
|
pcie_info = {
|
|
"tx_bytes": pcie_throughput_tx,
|
|
"rx_bytes": pcie_throughput_rx,
|
|
"generation": pcie_generation,
|
|
"width_max": pcie_width_max,
|
|
"width_current": pcie_width_current
|
|
}
|
|
except pynvml.NVMLError:
|
|
pcie_info = {
|
|
"tx_bytes": 0,
|
|
"rx_bytes": 0,
|
|
"generation": 0,
|
|
"width_max": 0,
|
|
"width_current": 0
|
|
}
|
|
|
|
# Error counters (ECC)
|
|
try:
|
|
if pynvml.nvmlDeviceGetEccMode(handle)[0]: # Check if ECC is enabled
|
|
volatile_ecc = pynvml.nvmlDeviceGetMemoryErrorCounter(handle, pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, pynvml.NVML_VOLATILE_ECC)
|
|
aggregate_ecc = pynvml.nvmlDeviceGetMemoryErrorCounter(handle, pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, pynvml.NVML_AGGREGATE_ECC)
|
|
ecc_errors = {
|
|
"enabled": True,
|
|
"volatile": volatile_ecc,
|
|
"aggregate": aggregate_ecc
|
|
}
|
|
else:
|
|
ecc_errors = {
|
|
"enabled": False,
|
|
"volatile": 0,
|
|
"aggregate": 0
|
|
}
|
|
except pynvml.NVMLError:
|
|
ecc_errors = {
|
|
"enabled": False,
|
|
"volatile": 0,
|
|
"aggregate": 0
|
|
}
|
|
|
|
gpu_info = {
|
|
"index": i,
|
|
"name": name,
|
|
"utilization": {
|
|
"gpu": gpu_util,
|
|
},
|
|
"memory": {
|
|
"total": round(memory_total, 2),
|
|
"used": round(memory_used, 2),
|
|
"percent": round(memory_percent, 2)
|
|
},
|
|
"temperature": temperature,
|
|
"power": {
|
|
"usage": round(power_usage, 2),
|
|
"limit": round(power_limit, 2)
|
|
},
|
|
"fan_speed": fan_speed,
|
|
"clocks": clock_info,
|
|
"pcie": pcie_info,
|
|
"ecc": ecc_errors
|
|
}
|
|
|
|
gpu_infos.append(gpu_info)
|
|
|
|
# Close NVML
|
|
pynvml.nvmlShutdown()
|
|
return {"gpus": gpu_infos, "timestamp": time.time()}
|
|
|
|
except Exception as e:
|
|
connect_print(f"Error retrieving GPU info: {str(e)}")
|
|
return {"error": str(e), "timestamp": time.time()}
|
|
|
|
def log_gpu_info(gpu_info):
|
|
"""Displays the most important GPU information in a condensed format"""
|
|
if "error" in gpu_info:
|
|
connect_print(f"GPU Info: Error: {gpu_info['error']}")
|
|
else:
|
|
for gpu in gpu_info["gpus"]:
|
|
# Condensed format: GPU_INDEX | NAME | UTIL% | MEM_USED/TOTAL (MEM%) | TEMP°C | POWER_USED/LIMIT W
|
|
connect_print(f"GPU {gpu['index']} | {gpu['name']} | Util: {gpu['utilization']['gpu']}% | Mem: {int(gpu['memory']['used'])}/{int(gpu['memory']['total'])}MB ({gpu['memory']['percent']}%) | Temp: {gpu['temperature']}°C | Power: {gpu['power']['usage']}/{gpu['power']['limit']}W | Fan: {gpu['fan_speed']}%") |