#!/usr/bin/env python3 # Copyright (c) Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT """ Check available AMD SMI metrics on the current system and output a GpuMetricAvailability struct per GPU. The struct fields mirror the PMC collector's initialize_supported_metrics() logic and can be used to filter validation rules in tests. Uses 'amd-smi metric --json' per category (-u, -t, -p, -P) for fine-grained sub-metric detection matching the C++ PMC collector. Uses 'amd-smi monitor --json' for mem_usage (separate API in C++). Uses 'amd-smi metric -x' for XGMI (only CLI source for XGMI presence). Usage: # Query live GPU(s) python3 check_amd_smi_metrics.py # JSON output python3 check_amd_smi_metrics.py --json """ import json import re import subprocess import sys from dataclasses import asdict, dataclass @dataclass class GpuMetricAvailability: """Per-GPU metric availability. Fine-grained fields mirror the C++ PMC collector's initialize_supported_metrics() in device.hpp. Coarse fields are derived for backward compatibility with ROCPROFSYS_AMD_SMI_METRICS.""" gpu_id: int = 0 # Fine-grained activity (from amd-smi metric -u --json) gfx_activity: bool = False # average_gfx_activity umc_activity: bool = False # average_umc_activity mm_activity: bool = False # average_mm_activity # Fine-grained temperature (from amd-smi metric -t --json) hotspot_temperature: bool = False # temperature_hotspot edge_temperature: bool = False # temperature_edge # Fine-grained power (from amd-smi metric -p --json) current_socket_power: bool = False # current_socket_power # Instinct per-XCP VCN/JPEG (from amd-smi metric -u --json) vcn_busy: bool = False # xcp_stats[].vcn_busy[] jpeg_busy: bool = False # xcp_stats[].jpeg_busy[] # Clocks (from amd-smi metric -c --json) gfx_clock: bool = False # current_gfxclk mem_clock: bool = False # current_uclk # Coarse fields (derived from fine-grained, kept for backward compat) busy: bool = False # gfx_activity or umc_activity or mm_activity temp: bool = False # hotspot_temperature or edge_temperature power: bool = False # current_socket_power mem_usage: bool = False # from amd-smi monitor (separate API in C++) vcn_activity: bool = False # device-level Radeon (mutually exclusive w/ vcn_busy) jpeg_activity: bool = False # device-level Radeon (mutually exclusive w/ jpeg_busy) xgmi: bool = False # from amd-smi metric -x (XGMI_ERR heuristic) pcie: bool = False # from amd-smi metric -P --json # sdma_usage is not reported by amd-smi CLI; it is a compile-time # feature (AMD_SMI_SDMA_SUPPORTED) and cannot be detected here. def to_metrics_string(self) -> str: """Return comma-separated string for ROCPROFSYS_AMD_SMI_METRICS.""" categories = [] for name in ( "busy", "temp", "power", "mem_usage", "xgmi", "pcie", ): if getattr(self, name): categories.append(name) if self.vcn_activity or self.vcn_busy: categories.append("vcn_activity") if self.jpeg_activity or self.jpeg_busy: categories.append("jpeg_activity") if self.gfx_clock: categories.append("gfx_clock") if self.mem_clock: categories.append("mem_clock") return ", ".join(categories) def _is_available(value) -> bool: """Return True if a JSON value represents an available metric. Handles: None, "N/A", {"value": N, "unit": "..."}, and plain numerics.""" if value is None: return False if isinstance(value, str): return value.strip() != "N/A" if isinstance(value, dict): # e.g. {"value": 25, "unit": "W"} - present means available return True # numeric (int, float) return True # --------------------------------------------------------------------------- # amd-smi command runners # --------------------------------------------------------------------------- def _run_command(cmd: list[str], exit_on_failure: bool = True) -> str | None: """Run a command and return stdout. If exit_on_failure is True (default), exits on error. If False, returns None on error (for graceful fallback). """ try: result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) if result.returncode != 0: if exit_on_failure: print( f"{' '.join(cmd)} failed (rc={result.returncode}):", file=sys.stderr, ) print(result.stderr, file=sys.stderr) sys.exit(1) return None return result.stdout except FileNotFoundError: if exit_on_failure: print("Error: 'amd-smi' not found in PATH", file=sys.stderr) sys.exit(1) return None except subprocess.TimeoutExpired: if exit_on_failure: print(f"Timeout running: {' '.join(cmd)}", file=sys.stderr) sys.exit(1) return None def _run_amd_smi_metric_json(flags: list[str]) -> list[dict]: """Run 'amd-smi metric --json' and return gpu_data list. Returns [] on any failure (graceful fallback). """ cmd = ["amd-smi", "metric"] + flags + ["--json"] stdout = _run_command(cmd, exit_on_failure=False) if stdout is None: return [] try: data = json.loads(stdout) return data.get("gpu_data", []) except (json.JSONDecodeError, AttributeError): return [] def run_amd_smi_monitor() -> list[dict]: """Run 'amd-smi monitor --json -w 1 -i 1' and return parsed JSON. Used only for mem_usage detection (C++ uses separate get_memory_usage() API). Returns [] on failure (graceful fallback). """ stdout = _run_command( ["amd-smi", "monitor", "-w", "1", "-i", "1", "--json"], exit_on_failure=False ) if stdout is None: return [] # Strip the leading info line ("'CTRL' + 'C' to stop watching output:") json_start = stdout.find("[") if json_start == -1: print("No JSON array found in amd-smi monitor output", file=sys.stderr) return [] try: return json.loads(stdout[json_start:]) except json.JSONDecodeError as exc: print(f"Failed to parse amd-smi monitor JSON: {exc}", file=sys.stderr) return [] # --------------------------------------------------------------------------- # Per-category detection from amd-smi metric --json # --------------------------------------------------------------------------- def _find_gpu_entry(gpu_data: list[dict], gpu_id: int) -> dict | None: """Find the entry matching gpu_id in gpu_data list.""" for entry in gpu_data: if entry.get("gpu") == gpu_id: return entry return None def detect_usage_metrics(gpus: list[GpuMetricAvailability]) -> None: """Detect activity metrics from 'amd-smi metric -u --json'. Populates: gfx_activity, umc_activity, mm_activity, vcn_activity, jpeg_activity, vcn_busy, jpeg_busy. Mirrors C++ initialize_supported_metrics() in device.hpp:293-354. """ gpu_data = _run_amd_smi_metric_json(["-u"]) if not gpu_data: return for gpu in gpus: entry = _find_gpu_entry(gpu_data, gpu.gpu_id) if entry is None: continue usage = entry.get("usage") if not isinstance(usage, dict): # GPU with "usage": "N/A" - no metrics available continue gpu.gfx_activity = _is_available(usage.get("gfx_activity")) gpu.umc_activity = _is_available(usage.get("umc_activity")) gpu.mm_activity = _is_available(usage.get("mm_activity")) # Per-XCP VCN/JPEG busy (Instinct) - check all XCP slots vcn_busy_data = usage.get("vcn_busy", {}) if isinstance(vcn_busy_data, dict): gpu.vcn_busy = any( _is_available(v) for xcp_vals in vcn_busy_data.values() if isinstance(xcp_vals, list) for v in xcp_vals ) jpeg_busy_data = usage.get("jpeg_busy", {}) if isinstance(jpeg_busy_data, dict): gpu.jpeg_busy = any( _is_available(v) for xcp_vals in jpeg_busy_data.values() if isinstance(xcp_vals, list) for v in xcp_vals ) # Device-level VCN/JPEG activity (Radeon) # C++ mutual exclusion: only set if per-XCP busy is NOT available vcn_activity_data = usage.get("vcn_activity", []) if isinstance(vcn_activity_data, list) and not gpu.vcn_busy: gpu.vcn_activity = any(_is_available(v) for v in vcn_activity_data) jpeg_activity_data = usage.get("jpeg_activity", []) if isinstance(jpeg_activity_data, list) and not gpu.jpeg_busy: gpu.jpeg_activity = any(_is_available(v) for v in jpeg_activity_data) def detect_temperature_metrics(gpus: list[GpuMetricAvailability]) -> None: """Detect temperature metrics from 'amd-smi metric -t --json'. Populates: hotspot_temperature, edge_temperature. """ gpu_data = _run_amd_smi_metric_json(["-t"]) if not gpu_data: return for gpu in gpus: entry = _find_gpu_entry(gpu_data, gpu.gpu_id) if entry is None: continue temp = entry.get("temperature") if not isinstance(temp, dict): continue gpu.hotspot_temperature = _is_available(temp.get("hotspot")) gpu.edge_temperature = _is_available(temp.get("edge")) def detect_power_metrics(gpus: list[GpuMetricAvailability]) -> None: """Detect power metrics from 'amd-smi metric -p --json'. Populates: current_socket_power. """ gpu_data = _run_amd_smi_metric_json(["-p"]) if not gpu_data: return for gpu in gpus: entry = _find_gpu_entry(gpu_data, gpu.gpu_id) if entry is None: continue power = entry.get("power") if not isinstance(power, dict): continue # amd-smi may expose current or average socket power under different keys. # The C++ collector treats them independently (device.hpp:308-311); # either one being valid causes device_power rows to be emitted. gpu.current_socket_power = _is_available( power.get("socket_power") ) or _is_available(power.get("average_socket_power")) def detect_clock_metrics(gpus: list[GpuMetricAvailability]) -> None: """Detect clock metrics from 'amd-smi metric -c --json'. Populates: gfx_clock, mem_clock. Mirrors C++ detection in device.hpp for current_gfxclk / current_uclk. The 'clock' section of `amd-smi metric -c --json` exposes per-domain sub-objects keyed like 'gfx_' and 'mem_'; presence of any non-N/A `clk` value within a domain group implies the metric is supported. """ gpu_data = _run_amd_smi_metric_json(["-c"]) if not gpu_data: return for gpu in gpus: entry = _find_gpu_entry(gpu_data, gpu.gpu_id) if entry is None: continue clock = entry.get("clock") if not isinstance(clock, dict): continue def _domain_available(prefix: str) -> bool: for key, val in clock.items(): if not key.startswith(prefix): continue if isinstance(val, dict) and _is_available(val.get("clk")): return True return False gpu.gfx_clock = _domain_available("gfx") gpu.mem_clock = _domain_available("mem") def detect_pcie_metrics(gpus: list[GpuMetricAvailability]) -> None: """Detect PCIe metrics from 'amd-smi metric -P --json'. Populates: pcie. Mirrors C++ OR logic in device.hpp:363-367. """ gpu_data = _run_amd_smi_metric_json(["-P"]) if not gpu_data: return for gpu in gpus: entry = _find_gpu_entry(gpu_data, gpu.gpu_id) if entry is None: continue pcie = entry.get("pcie") if not isinstance(pcie, dict): continue # Mirrors C++ OR logic at device.hpp:363-367 gpu.pcie = ( _is_available(pcie.get("width")) or _is_available(pcie.get("speed")) or _is_available(pcie.get("bandwidth")) or _is_available(pcie.get("current_bandwidth_sent")) or _is_available(pcie.get("current_bandwidth_received")) ) # --------------------------------------------------------------------------- # XGMI detection (kept from original - text parsing of amd-smi metric -x) # --------------------------------------------------------------------------- def _parse_xgmi_metric(text: str) -> dict[int, bool]: """Parse 'amd-smi metric -x' output. Returns {gpu_id: xgmi_available}.""" result: dict[int, bool] = {} current_gpu = None for line in text.splitlines(): gpu_match = re.match(r"^GPU:\s*(\d+)", line.strip()) if gpu_match: current_gpu = int(gpu_match.group(1)) continue if current_gpu is not None: m = re.match(r"^\s+XGMI_ERR:\s*(.+)$", line) if m: result[current_gpu] = m.group(1).strip() != "N/A" current_gpu = None return result def run_amd_smi_xgmi() -> dict[int, bool]: """Run 'amd-smi metric -x' and return {gpu_id: xgmi_available}.""" stdout = _run_command(["amd-smi", "metric", "-x"], exit_on_failure=False) if stdout is None: return {} return _parse_xgmi_metric(stdout) # --------------------------------------------------------------------------- # Monitor-based detection (kept for mem_usage only) # --------------------------------------------------------------------------- # Only mem_usage uses monitor - it maps to a separate C++ API (get_memory_usage) _MONITOR_FIELD_MAP: dict[str, list[str]] = { "mem_usage": ["vram_used", "vram_total"], } def parse_monitor_json(data: list[dict]) -> list[GpuMetricAvailability]: """Parse 'amd-smi monitor --json' output into per-GPU availability structs. Only populates mem_usage. Other fields come from amd-smi metric --json. """ gpus: list[GpuMetricAvailability] = [] for entry in data: gpu = GpuMetricAvailability(gpu_id=entry.get("gpu", 0)) for field, keys in _MONITOR_FIELD_MAP.items(): available = any(_is_available(entry.get(k)) for k in keys) setattr(gpu, field, available) gpus.append(gpu) return gpus # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- # Names emitted by get_available_metrics_set / get_available_metrics_per_gpu _FINE_GRAINED_NAMES = ( "gfx_activity", "umc_activity", "mm_activity", "hotspot_temperature", "edge_temperature", "current_socket_power", "vcn_busy", "jpeg_busy", "gfx_clock", "mem_clock", ) # Note: vcn_activity and jpeg_activity are handled separately in # collect_metric_names() because they can be sourced from two C++ paths # (device-level Radeon vs per-XCP MI300). _COARSE_NAMES = ( "busy", "temp", "power", "mem_usage", "xgmi", "pcie", ) def _build_gpu_list_from_metric(gpu_data: list[dict]) -> list[GpuMetricAvailability]: """Build GPU list from amd-smi metric output (fallback when monitor fails).""" return [ GpuMetricAvailability(gpu_id=entry.get("gpu", i)) for i, entry in enumerate(gpu_data) ] def get_available_metrics() -> list[GpuMetricAvailability]: """Detect available metrics on all GPUs. Importable entry point.""" # 1. Get GPU list + mem_usage from monitor monitor_data = run_amd_smi_monitor() gpus = parse_monitor_json(monitor_data) # Fallback: if monitor failed, build GPU list from metric -u output if not gpus: metric_data = _run_amd_smi_metric_json(["-u"]) gpus = _build_gpu_list_from_metric(metric_data) # 2. Fine-grained detection via amd-smi metric (each gracefully falls back) detect_usage_metrics(gpus) detect_temperature_metrics(gpus) detect_power_metrics(gpus) detect_pcie_metrics(gpus) detect_clock_metrics(gpus) # 3. XGMI from existing text parser xgmi_map = run_amd_smi_xgmi() for gpu in gpus: gpu.xgmi = xgmi_map.get(gpu.gpu_id, False) # 4. Derive coarse fields from fine-grained for gpu in gpus: gpu.busy = gpu.gfx_activity or gpu.umc_activity or gpu.mm_activity gpu.temp = gpu.hotspot_temperature or gpu.edge_temperature gpu.power = gpu.current_socket_power return gpus def collect_metric_names(gpu: GpuMetricAvailability) -> set[str]: """Collect all available metric names (fine-grained + coarse) for a GPU.""" metrics: set[str] = set() for name in _FINE_GRAINED_NAMES: if getattr(gpu, name): metrics.add(name) for name in _COARSE_NAMES: if getattr(gpu, name): metrics.add(name) # vcn_activity/jpeg_activity is the user-facing metric name that covers # both device-level (Radeon) and per-XCP (MI300) paths. if gpu.vcn_activity or gpu.vcn_busy: metrics.add("vcn_activity") if gpu.jpeg_activity or gpu.jpeg_busy: metrics.add("jpeg_activity") return metrics def get_available_metrics_set() -> set[str]: """Return the union of available metric names across all GPUs.""" gpus = get_available_metrics() metrics: set[str] = set() for gpu in gpus: metrics |= collect_metric_names(gpu) return metrics def get_available_metrics_per_gpu() -> dict[int, set[str]]: """Return a dict mapping GPU ID to its available metric names.""" gpus = get_available_metrics() return {gpu.gpu_id: collect_metric_names(gpu) for gpu in gpus} # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def main(): json_output = "--json" in sys.argv gpus = get_available_metrics() if not gpus: print("No GPUs found in amd-smi output", file=sys.stderr) sys.exit(1) if json_output: print(json.dumps([asdict(g) for g in gpus], indent=2)) else: for gpu in gpus: print(f"GPU {gpu.gpu_id}:") print(f" gfx_activity: {gpu.gfx_activity}") print(f" umc_activity: {gpu.umc_activity}") print(f" mm_activity: {gpu.mm_activity}") print(f" hotspot_temperature: {gpu.hotspot_temperature}") print(f" edge_temperature: {gpu.edge_temperature}") print(f" current_socket_power: {gpu.current_socket_power}") print(f" vcn_activity: {gpu.vcn_activity}") print(f" jpeg_activity: {gpu.jpeg_activity}") print(f" vcn_busy: {gpu.vcn_busy}") print(f" jpeg_busy: {gpu.jpeg_busy}") print(f" gfx_clock: {gpu.gfx_clock}") print(f" mem_clock: {gpu.mem_clock}") print(f" mem_usage: {gpu.mem_usage}") print(f" xgmi: {gpu.xgmi}") print(f" pcie: {gpu.pcie}") print(f" ---") print(f" busy (derived): {gpu.busy}") print(f" temp (derived): {gpu.temp}") print(f" power (derived): {gpu.power}") print(f' -> ROCPROFSYS_AMD_SMI_METRICS="{gpu.to_metrics_string()}"') print() if __name__ == "__main__": main()