import { existsSync } from "node:fs"; import type { GpuInfo, RuntimeGpuMonitoringTool } from "../../models/types"; import { runCommand } from "../../../core/command"; import { getGpuInfoFromAmdSmi, getGpuInfoFromRocmSmi } from "./amd-gpu"; import { getGpuInfoFromIntelSysfs } from "./intel-gpu"; import { resolveRocmSmiTool } from "./rocm-info"; import { resolveAmdSmiBinary, resolveForcedGpuMonitoringTool, resolveNvidiaSmiBinary, resolveRocmSmiBinary, } from "./smi-tools"; export const getGpuInfoFromNvidiaSmi = (): GpuInfo[] => { const query = [ "name", "memory.total", "memory.used", "memory.free", "utilization.gpu", "temperature.gpu", "power.draw", "power.limit", ].join(","); try { const nvidiaSmi = resolveNvidiaSmiBinary(); if (!nvidiaSmi) return []; const result = runCommand( nvidiaSmi, [`--query-gpu=${query}`, "--format=csv,noheader,nounits"], 5_000 ); if (result.status !== 0 || !result.stdout) return []; const lines = result.stdout .trim() .split("\n") .map((line) => line.trim()) .filter(Boolean); return lines.map((line, index) => { const parts = line.split(",").map((value) => value.trim()); const [ name, memoryTotal, memoryUsed, memoryFree, utilization, temperature, powerDraw, powerLimit, ] = parts; const toBytes = (megabytes: string | undefined): number => Math.max(0, Math.round(Number(megabytes ?? 0) * 1024 * 1024)); const toMb = (megabytes: string | undefined): number => Math.max(0, Math.round(Number(megabytes ?? 0))); return { index, name: name ?? "Unknown", memory_total: toBytes(memoryTotal), memory_total_mb: toMb(memoryTotal), memory_used: toBytes(memoryUsed), memory_used_mb: toMb(memoryUsed), memory_free: toBytes(memoryFree), memory_free_mb: toMb(memoryFree), utilization: Number(utilization ?? 0), utilization_pct: Number(utilization ?? 0), temperature: Number(temperature ?? 0), temp_c: Number(temperature ?? 0), power_draw: Number(powerDraw ?? 0), power_limit: Number(powerLimit ?? 0), }; }); } catch { return []; } }; /** Tool the cascade in getGpuInfo would use, without running any query commands. */ export const detectGpuMonitoringTool = (): RuntimeGpuMonitoringTool | null => { const forced = resolveForcedGpuMonitoringTool(); if (forced) return forced; if (resolveNvidiaSmiBinary()) return "nvidia-smi"; const rocmTool = resolveRocmSmiTool(); if (rocmTool) return rocmTool; if (getGpuInfoFromIntelSysfs().length > 0) return "intel-sysfs"; return null; }; // Logged once per process so CPU-only and missing-driver hosts are distinguishable // from "zero GPUs" without spamming every poll. let warnedNoGpuTooling = false; const warnNoGpuToolingOnce = (): void => { if (warnedNoGpuTooling) return; warnedNoGpuTooling = true; const attempted = [ `nvidia-smi=${resolveNvidiaSmiBinary() ? "found" : "not found"}`, `amd-smi=${resolveAmdSmiBinary() ? "found" : "not found"}`, `rocm-smi=${resolveRocmSmiBinary() ? "found" : "not found"}`, `intel-sysfs=${existsSync("/sys/bus/pci/devices") ? "no compute GPUs" : "unavailable"}`, ].join(" "); console.warn(`No GPUs reported by any monitoring tool; attempted: ${attempted}`); }; const collectGpuInfo = (): GpuInfo[] => { const forced = resolveForcedGpuMonitoringTool(); if (forced === "nvidia-smi") { return getGpuInfoFromNvidiaSmi(); } if (forced === "amd-smi") { return getGpuInfoFromAmdSmi(); } if (forced === "rocm-smi") { return getGpuInfoFromRocmSmi(); } if (forced === "intel-sysfs") { return getGpuInfoFromIntelSysfs(); } const nvidia = getGpuInfoFromNvidiaSmi(); if (nvidia.length > 0) { return nvidia; } const rocmTool = resolveRocmSmiTool(); if (rocmTool === "amd-smi") { const amd = getGpuInfoFromAmdSmi(); if (amd.length > 0) return amd; return getGpuInfoFromRocmSmi(); } if (rocmTool === "rocm-smi") { const rocm = getGpuInfoFromRocmSmi(); if (rocm.length > 0) return rocm; return getGpuInfoFromAmdSmi(); } const intel = getGpuInfoFromIntelSysfs(); if (intel.length > 0) { return intel; } return []; }; export const getGpuInfo = (): GpuInfo[] => { const gpus = collectGpuInfo(); if (gpus.length === 0) { warnNoGpuToolingOnce(); } return gpus; };