TheRock/rocm-systems/projects/rocprofiler-systems/tests/pytest/rocprofsys/gpu.py

# Copyright (c) Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

from __future__ import annotations
import re
import shutil
import subprocess
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Optional

from .cache import persistent_cache

GFX_XXXX_PATTERN = re.compile(r"(gfx[0-9a-fA-F]+)")

# Architectures where GPU perf counter (device counting service) validation is
# unreliable and dependent tests should be skipped.
UNSUPPORTED_PERF_COUNTER_GFX = frozenset(
    {"gfx1101", "gfx1102", "gfx1150", "gfx1151", "gfx1152", "gfx1153"}
)


@dataclass
class GPUInfo:
    """Information about detected GPU(s)

    Attributes:
        available: Whether any GPU is available
        architectures: List of GPU architectures
        device_count: Number of GPUs detected
        categories: Categories the GPU belongs to (instinct, radeon, apu)
    """

    available: bool
    architectures: list[str]
    device_count: int
    categories: set[str]

    @property
    def _is_mi300_or_later(self) -> bool:
        """Check if the GPU is a MI300 or later."""
        for arch in self.architectures:
            if re.match(r"gfx9[4-9][0-9A-Fa-f]", arch):
                return True
        return False

    @property
    def _is_gfx1250(self) -> bool:
        """Check if any detected GPU uses the gfx1250 architecture."""
        return "gfx1250" in self.architectures

    @property
    def rocm_events_for_test(self) -> str:
        """Get appropriate ROCm events for testing based on architecture."""
        if self._is_gfx1250:
            return "GRBM_COUNT,SQ_WAVES,SQ_INSTS_VALU,TX_VCA_VCA_BUSY"

        mi300_or_later = self._is_mi300_or_later
        if mi300_or_later:
            return "GRBM_COUNT,SQ_WAVES,SQ_INSTS_VALU,TA_TA_BUSY"
        return "SQ_WAVES"

    @property
    def counter_names(self) -> list[str]:
        """Get counter names for validation based on architecture"""
        if self._is_gfx1250:
            return ["GRBM_COUNT", "SQ_WAVES", "SQ_INSTS_VALU", "TX_VCA_VCA_BUSY"]

        mi300_or_later = self._is_mi300_or_later
        if mi300_or_later:
            return ["GRBM_COUNT", "SQ_WAVES", "SQ_INSTS_VALU", "TA_TA_BUSY"]
        return ["SQ_WAVES"]

    @property
    def gpu_perf_counters_for_test(self) -> str:
        """Get appropriate GPU perf counters for testing based on architecture.

        These are the same counters as rocm_events_for_test but used with
        ROCPROFSYS_GPU_PERF_COUNTERS (device counting service) instead of
        ROCPROFSYS_ROCM_EVENTS (kernel dispatch counters).
        """
        return self.rocm_events_for_test

    @property
    def expected_counter_files(self) -> list[str]:
        """Get expected counter output file patterns based on architecture.

        Returns glob patterns that match any device ID (0-9), since the device
        number in the filename depends on device_type_index which varies by
        GPU topology. The optional trailing suffix matches per-output indices
        such as ``rocprof-device-0-SQ_WAVES-0.txt``.
        """
        return [f"rocprof-device-[0-9]-{name}*.txt" for name in self.counter_names]

    @property
    def unsupported_perf_counter_archs(self) -> set[str]:
        """Detected architectures where GPU perf counter validation is unreliable.

        Returns the intersection of the detected architectures with
        ``UNSUPPORTED_PERF_COUNTER_GFX``.
        https://github.com/ROCm/rocm-systems/blob/develop/projects/rocprofiler-sdk/tests/rocprofv3/counter-collection/input1/validate.py#L83C4-L83C82
        """
        return set(UNSUPPORTED_PERF_COUNTER_GFX.intersection(self.architectures))


def get_rocminfo(rocm_path: Optional[Path] = None) -> Optional[Path]:
    """Get the path to the rocminfo executable.

    Args:
        rocm_path: Path to the ROCm installation directory

    Returns:
        Path to the rocminfo executable or None if not found
    """
    if rocm_path:
        candidate = rocm_path / "bin" / "rocminfo"
        if candidate.exists():
            return Path(candidate).resolve()
    rocminfo = shutil.which("rocminfo")
    if rocminfo:
        return Path(rocminfo).resolve()
    return None


# rocminfo can fail for several reasons, including having no GPU devices.
# A GPU-less machine is a stable, expected state, so it is treated as "no GPU".
# Any other failure (kfd perms, kfd/HSA-init races) is treated as an error.
# This marker matches only the GPU-less case; see rocminfo.cc CheckInitialState().
_ROCMINFO_NO_GPU_MARKER = "possibly no gpu devices"


def _rocminfo_reports_no_gpu(output: str) -> bool:
    """Return True if rocminfo output indicates the system has no AMD GPU."""
    return _ROCMINFO_NO_GPU_MARKER in output.lower()


def run_rocminfo(rocm_path: Optional[Path] = None) -> Optional[str]:
    """Run ``rocminfo`` and return its stdout for further processing.

    Args:
        rocm_path: Path to the ROCm installation directory

    Returns:
        The captured stdout on success, or ``None`` if ``rocminfo`` cannot be
        found or the system has no AMD GPU (ROCk kernel module not loaded).

    Raises:
        RuntimeError: if ``rocminfo`` is found but fails for any reason other
            than "no GPU present" (non-zero exit, timeout, or OS error)
    """
    rocminfo = get_rocminfo(rocm_path)
    if not rocminfo:
        return None
    try:
        result = subprocess.run(
            [str(rocminfo)],
            capture_output=True,
            text=True,
            timeout=30,
        )
    except (subprocess.TimeoutExpired, OSError) as exc:
        raise RuntimeError(f"Failed to run rocminfo ({rocminfo}): {exc}") from exc

    if result.returncode != 0:
        # A system with no AMD GPU is a valid, stable state, not an error
        if _rocminfo_reports_no_gpu(result.stdout + result.stderr):
            return None
        raise RuntimeError(
            f"rocminfo ({rocminfo}) failed with return code {result.returncode}:\n"
            f"--- stdout ---\n{result.stdout.strip()}\n"
            f"--- stderr ---\n{result.stderr.strip()}"
        )
    return result.stdout


@persistent_cache("gpu.detect_gpu")
def detect_gpu(rocm_path: Optional[Path] = None) -> GPUInfo:
    """Detect available AMD GPUs and their capabilities.

    Uses rocminfo to get the list of GPU architectures.
    Regex avoids matching "gfxX-X-generic" which may appear.
    """
    categories: set[str] = set()
    architectures: list[str] = []
    device_count = 0

    rocminfo_stdout = run_rocminfo(rocm_path)
    if rocminfo_stdout:
        # Only match gfx on "Name:"
        name_gfx_pattern = re.compile(
            r"^\s*Name:\s+(gfx[0-9A-Fa-f][0-9A-Fa-f]+)", re.MULTILINE
        )
        all_matches = name_gfx_pattern.findall(rocminfo_stdout)
        # gfx000 is the cpu, remove it
        filtered = [arch for arch in all_matches if arch != "gfx000"]
        device_count = len(filtered)
        # Remove duplicates
        architectures = list(set(filtered))

    for arch in architectures:
        categories.update(lookup_gpu_category(arch, rocminfo_stdout))

    return GPUInfo(
        available=device_count > 0,
        architectures=sorted(architectures),
        device_count=device_count,
        categories=categories,
    )


def lookup_gpu_category(
    arch: str,
    rocminfo_stdout: Optional[str] = None,
) -> list[str]:
    """Lookup the GPU category for an architecture.

    Args:
        arch: Architecture string (e.g., 'gfx940')
        rocminfo_stdout: Pre-captured rocminfo stdout, used for the APU check

    Returns:
        List of GPU categories the architecture belongs to (instinct, radeon, apu)
    """
    instinct_list = [
        "gfx900",
        "gfx906",  # MI50/MI60
        "gfx908",
        "gfx90a",
        "gfx942",
        "gfx950",
    ]

    # Also includes PRO GPUs
    # Ignore Radeon VII (gfx906)
    radeon_list = [
        "gfx1010",
        "gfx1011",
        "gfx1012",
        "gfx1030",
        "gfx1031",
        "gfx1032",
        "gfx1100",
        "gfx1101",
        "gfx1102",
        "gfx1200",
        "gfx1201",
        "gfx1202",
    ]

    apu_list = [
        "gfx1035",
        "gfx1036",
        "gfx1103",
        "gfx1151",
        "gfx1152",
        "gfx1153",
    ]

    categories: list[str] = []

    if arch in instinct_list:
        categories.append("instinct")
        # Some instinct GPUs may also be an APU (ex: MI300A).
        if rocminfo_stdout and "APU" in rocminfo_stdout:
            categories.append("apu")
    if arch in radeon_list:
        categories.append("radeon")
    if arch in apu_list:
        categories.append("apu")

    if not categories:
        # Unknown architecture, default to instinct
        categories.append("instinct")

    return categories


@persistent_cache("gpu.get_offload_extractor")
def get_offload_extractor(
    rocm_path: Optional[Path] = None,
) -> tuple[Optional[Path], Optional[bool]]:
    """Get offload extractor path

    An offload extractor is one of:
        llvm-objdump (only if version >= 20) - Preferred
        roc-obj-ls (deprecated)              - Fallback

    Args:
        rocm_path: Path to the ROCm installation directory

    Returns:
        Path to the offload extractor
        Bool representing whether found llvm-objdump's version < 20 (None if llvm-objdump not found)
    """

    is_llvm_too_old = None
    offload_extractor = None
    # Check env var - accepts either path to binary or directory containing it
    llvm_objdump_env = os.environ.get("ROCM_LLVM_OBJDUMP")
    if llvm_objdump_env:
        llvm_objdump_path = Path(llvm_objdump_env)
        if llvm_objdump_path.is_file() and llvm_objdump_path.exists():
            offload_extractor = llvm_objdump_path
        elif llvm_objdump_path.is_dir():
            candidate = llvm_objdump_path / "llvm-objdump"
            if candidate.exists():
                offload_extractor = candidate

    # Fallback to ROCm path
    if not offload_extractor and rocm_path:
        llvm_objdump_candidates = [
            rocm_path / "llvm" / "bin" / "llvm-objdump",
            rocm_path / "bin" / "llvm-objdump",
        ]
        for candidate in llvm_objdump_candidates:
            if candidate.exists():
                offload_extractor = candidate
                break

    if offload_extractor:
        # We have found llvm-objdump, check version
        try:
            version_result = subprocess.run(
                [str(offload_extractor), "--version"],
                capture_output=True,
                text=True,
                timeout=10,
            )
            if version_result.returncode == 0:
                version_match = re.search(r"version\s+(\d+)", version_result.stdout or "")
                if version_match:
                    major_version = int(version_match.group(1))
                    if major_version >= 20:
                        is_llvm_too_old = False
                        return (
                            Path(offload_extractor).resolve(),
                            is_llvm_too_old,
                        )
                    else:
                        is_llvm_too_old = True
        except (subprocess.TimeoutExpired, OSError, ValueError):
            pass

    # Fallback to roc-obj-ls
    offload_extractor = None
    if rocm_path:
        candidate = rocm_path / "bin" / "roc-obj-ls"
        if candidate.exists():
            offload_extractor = Path(candidate).resolve()
            return offload_extractor, is_llvm_too_old
    if not offload_extractor:
        offload_extractor = shutil.which("roc-obj-ls")
    if offload_extractor:
        return Path(offload_extractor).resolve(), is_llvm_too_old
    return None, is_llvm_too_old


@persistent_cache("gpu.get_target_gpu_arch")
def get_target_gpu_arch(rocm_path: Path, target_path: Path) -> list[str]:
    """Get the list of gpu architectures (gfx) the target was compiled for.

    Cached per ``(rocm_path, target_path)`` for the CTest session (a target
    binary does not change mid-session; the config-setup fixture regenerates
    the cache between runs).

    Args:
        rocm_path: Path to the ROCm installation directory
        target_path: Path to the binary to check

    Returns:
        List of GPU architectures the target was compiled for

    Raises:
        FileNotFoundError: If offload extractor is not found
    """
    import tempfile

    target_archs: set[str] = set()

    result = get_offload_extractor(rocm_path)
    tool_path, _ = result
    if not tool_path:
        raise FileNotFoundError(
            f"Could not find offload extractor in {rocm_path} "
            "or environment variable ROCM_LLVM_OBJDUMP"
        )

    if "llvm-objdump" in tool_path.name:
        with tempfile.TemporaryDirectory() as tmpdir:
            tmp_symlink = Path(tmpdir) / target_path.name
            try:
                tmp_symlink.symlink_to(target_path)
            except OSError:
                return list(target_archs)

            extracted_files: list[Path] = []
            try:
                result = subprocess.run(
                    [str(tool_path), "--offloading", str(tmp_symlink)],
                    capture_output=True,
                    text=True,
                    timeout=30,
                )
                if result.returncode == 0:
                    for line in result.stdout.strip().split("\n"):
                        match = GFX_XXXX_PATTERN.search(line)
                        if match:
                            target_archs.add(match.group(1))

                        # Capture extracted bundle paths for cleanup
                        bundle_match = re.search(
                            r"Extracting offload bundle:\s*(.+)$", line
                        )
                        if bundle_match:
                            extracted_files.append(Path(bundle_match.group(1)))
            except (subprocess.TimeoutExpired, OSError):
                pass

            # Immediately clean up extracted files to free disk space
            for extracted_file in extracted_files:
                try:
                    if extracted_file.exists():
                        extracted_file.unlink()
                except OSError:
                    pass

    elif "roc-obj-ls" in tool_path.name:
        try:
            result = subprocess.run(
                [str(tool_path), str(target_path)],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode == 0:
                for line in result.stdout.strip().split("\n"):
                    match = GFX_XXXX_PATTERN.search(line)
                    if match:
                        target_archs.add(match.group(1))
        except (subprocess.TimeoutExpired, OSError):
            pass

    return list(target_archs)


@persistent_cache("gpu.get_xnack_support")
def get_xnack_support(rocm_path: Optional[Path] = None) -> bool:
    """Check whether the current GPU is XNACK-capable.

    Run ``rocminfo`` with ``HSA_XNACK=1`` injected into the subprocess
    environment and return True only if the output reports ``xnack+``.

    This keeps the check independent of the caller's shell environment:
    ``xnack-`` remains unsupported for test gating, and GPUs with no
    XNACK qualifier also return False.
    """
    rocminfo = get_rocminfo(rocm_path)
    if not rocminfo:
        return False

    try:
        env = os.environ.copy()
        env["HSA_XNACK"] = "1"
        result = subprocess.run(
            [str(rocminfo)],
            capture_output=True,
            text=True,
            timeout=30,
            env=env,
        )
        if result.returncode == 0:
            return "xnack+" in result.stdout
    except (subprocess.TimeoutExpired, OSError):
        pass

    return False