derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
2860 lines
105 KiB
Python
2860 lines
105 KiB
Python
# Copyright (c) Advanced Micro Devices, Inc.
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
"""
|
|
Pytest configuration and fixtures for rocprofiler-systems tests.
|
|
|
|
This module provides shared fixtures and configuration for all test modules.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
from pathlib import Path
|
|
from functools import lru_cache
|
|
from typing import Any, Callable, Generator, Optional
|
|
|
|
import re
|
|
import os
|
|
import sys
|
|
import shutil
|
|
import signal
|
|
import traceback
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError: # pragma: no cover
|
|
yaml = None # type: ignore[assignment]
|
|
|
|
# Add the pytest directory to Python path for rocprofsys package
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
import pytest
|
|
from pytest import StashKey
|
|
|
|
from rocprofsys import environment
|
|
from rocprofsys import (
|
|
RocprofsysConfig,
|
|
discover_build_config,
|
|
GPUInfo,
|
|
get_rocminfo,
|
|
detect_gpu,
|
|
get_offload_extractor,
|
|
get_target_gpu_arch,
|
|
get_xnack_support,
|
|
TestResult,
|
|
ValidationResult,
|
|
validate_regex,
|
|
validate_file_regex,
|
|
validate_perfetto_trace,
|
|
validate_rocpd_database,
|
|
validate_timemory_json,
|
|
validate_causal_json,
|
|
validate_unified_memory_outputs,
|
|
validate_file_exists,
|
|
BaselineRunner,
|
|
SamplingRunner,
|
|
BinaryRewriteRunner,
|
|
RuntimeInstrumentRunner,
|
|
SysRunRunner,
|
|
CausalRunner,
|
|
PythonRunner,
|
|
safe_remove,
|
|
)
|
|
|
|
# Key for storing the single test result on pytest items
|
|
# Item-level stash keys
|
|
_result_key: StashKey = StashKey()
|
|
_output_printed_key: StashKey[bool] = StashKey()
|
|
_original_nodeid_key: StashKey[str] = StashKey()
|
|
|
|
# GNU convention. Used for CTests
|
|
SKIP_RETURN_CODE = 77
|
|
|
|
# As test + subtests are collapsed into a single pytest, there is only one global timeout that the
|
|
# ctest will use as reference.
|
|
# The DEFAULT_TIMEOUT is used as the "run_test" timeout and
|
|
# the CTEST_TIMEOUT_BUFFER is a fixed amount added to handle subtests + flush + teardown
|
|
# CTests set their timeout to DEFAULT_TIMEOUT + CTEST_TIMEOUT_BUFFER
|
|
DEFAULT_TIMEOUT = 300
|
|
CTEST_TIMEOUT_BUFFER = 30 # Not overridable
|
|
|
|
# Timeouts for the "rocprofiler-systems-pytest-config" prerequisite test
|
|
CONFIG_TIMEOUT = 15
|
|
CONFIG_TIMEOUT_BUFFER = 5
|
|
|
|
# Accepted runner types when using parametrized "mode" marker
|
|
ROCPROFSYS_RUNNER_CLASSES = {
|
|
"baseline": BaselineRunner,
|
|
"sampling": SamplingRunner,
|
|
"binary_rewrite": BinaryRewriteRunner,
|
|
"runtime_instrument": RuntimeInstrumentRunner,
|
|
"sys_run": SysRunRunner,
|
|
"causal": CausalRunner,
|
|
"python": PythonRunner,
|
|
}
|
|
# Accepted runner types when using parametrized "mode" marker
|
|
ROCPROFSYS_RUNNER_NAMES = list(ROCPROFSYS_RUNNER_CLASSES.keys())
|
|
|
|
# ============================================================================
|
|
#
|
|
# Pytest Hooks (Placed in the general order they are called)
|
|
#
|
|
# ============================================================================
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Initialization hooks
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
|
"""Add custom command-line options."""
|
|
group = parser.getgroup("rocprofsys", "rocprofiler-systems test options")
|
|
group.addoption(
|
|
"--show-config-only", # Only used by "rocprofiler-systems-pytest-config" test
|
|
action="store_true",
|
|
default=False,
|
|
help="Show the test configuration and exit without running any tests",
|
|
)
|
|
group.addoption(
|
|
"--ctest-mode",
|
|
action="store",
|
|
default="off",
|
|
choices=("off", "generate", "run", "cleanup"),
|
|
help="CTest integration mode (developer flag): 'off' (default), 'generate', 'run', or 'cleanup'",
|
|
)
|
|
group.addoption(
|
|
"--ctest-output-path",
|
|
action="store",
|
|
default=None,
|
|
help="Path to write the CTest definitions file when in CTest generate mode (default: None)",
|
|
)
|
|
group.addoption(
|
|
"--python-versions",
|
|
action="store",
|
|
default=None,
|
|
help="Semicolon-separated list of Python versions (e.g. '3.8;3.9;3.10')",
|
|
)
|
|
group.addoption(
|
|
"--python-root-dirs",
|
|
action="store",
|
|
default=None,
|
|
help="Semicolon-separated list of directories to search for Python interpreters in order of preference",
|
|
)
|
|
|
|
|
|
def pytest_configure(config: pytest.Config) -> None:
|
|
"""Register custom markers and configure pytest"""
|
|
|
|
config.option.verbose = max(config.option.verbose, 1) # -v
|
|
config.option.tbstyle = "short" # --tb=short
|
|
config.option.no_header = True
|
|
config.option.reportchars += "s" # -rs
|
|
|
|
# CTest generation only collects tests to emit CTestTestfile.cmake
|
|
if config.getoption("--ctest-mode", default="off") == "generate":
|
|
from rocprofsys.cache import disable_for_process
|
|
|
|
disable_for_process()
|
|
|
|
if config.getoption("--ctest-mode", default="off") == "cleanup":
|
|
_run_cleanup()
|
|
pytest.exit("Cleanup complete", returncode=0)
|
|
|
|
if config.getoption("--show-config-only", default=False):
|
|
pytest._config_ref = config
|
|
_seed_capability_cache()
|
|
header = _generate_rocprofsys_config_header()
|
|
for line in header:
|
|
print(line)
|
|
pytest.exit("Header generated", returncode=0)
|
|
|
|
# Disable pytest-timeout plugin if detected
|
|
# It will interfere with our timeout marker
|
|
timeout_plugin = config.pluginmanager.get_plugin("timeout")
|
|
if timeout_plugin:
|
|
config.pluginmanager.unregister(timeout_plugin)
|
|
|
|
# Functional markers (do more than just label a test)
|
|
# See pytest_collection_modifyitems
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"gpu: mark test as requiring a GPU",
|
|
) # required for run_test to check if the target supports the current system architectures
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"mpi_optional(target): If MPI is available and the target supports MPI, uses MPI to run the test",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"preserve(file): prevents the file from being deleted after the test, even if ROCPROFSYS_KEEP_TEST_OUTPUT is set to OFF",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"run_if_gpu_category(expr): run test only if GPU category expression is true "
|
|
"(e.g., 'apu and not instinct', 'instinct or radeon')",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"rocm_min_version(version): mark test as requiring minimum ROCm version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"oshrun_min_version(version): mark test as requiring minimum OpenSHMEM version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"amdsmi_min_version(version): mark test as requiring minimum AMD-SMI version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"amdgpu_min_version(version): mark test as requiring minimum amdgpu driver version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"rocprofiler_sdk_min_version(version): mark test as requiring minimum rocprofiler-sdk version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"rocpd(env): mark test as using ROCpd and inject ROCpd env into given env",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"mpi_implementation(implementation): mark test as requiring specific MPI implementation",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"python_versions: Test will be parametrized by Python version",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"timeout(seconds): mark test as having a timeout of seconds (default: 300)",
|
|
)
|
|
# Used for CTest
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"depends_on(*names): declare CTest dependency on the named tests"
|
|
" (used for CTest DEPENDS)",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"serialize: mark test as serializable (used for CTest)",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"class_name(segment): hyphenated logical name for standardized / CTest test "
|
|
"names (replaces the auto-derived class segment from TestCamelCase; "
|
|
"e.g. 'rocprofiler-systems-instrument')",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"multi_gpu(num): mark test as using requiring atleast num amount of GPUs",
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"build_only: prevents the test from being run in install mode", # TheRock CI runs in install mode
|
|
)
|
|
|
|
# See pytest_collection_modifyitems
|
|
generic_functional_markers = [
|
|
"ucx",
|
|
"overflow",
|
|
"attach",
|
|
"mpi",
|
|
"python",
|
|
"annotate",
|
|
"julia",
|
|
"xnack",
|
|
"no_docker",
|
|
"shmem",
|
|
"nic",
|
|
"ainic_required",
|
|
]
|
|
|
|
# Informational markers, only used for test labeling
|
|
|
|
config.addinivalue_line(
|
|
"markers", "rocprofiler: mark test as using ROCProfiler counters"
|
|
)
|
|
config.addinivalue_line("markers", "slow: mark test as slow running")
|
|
config.addinivalue_line("markers", "loops: mark test as testing loop instrumentation")
|
|
|
|
# Can be described using generic desc below
|
|
non_functional_markers = [
|
|
"avail",
|
|
"instrument",
|
|
"baseline",
|
|
"sampling",
|
|
"binary_rewrite",
|
|
"runtime_instrument",
|
|
"sys_run",
|
|
"decode",
|
|
"videodecode",
|
|
"jpegdecode",
|
|
"hipfile",
|
|
"rocprof_binary",
|
|
"rocprof_config",
|
|
"xgmi",
|
|
"sdma",
|
|
"group_by_queue",
|
|
"group_by_stream",
|
|
"openmp",
|
|
"openmp_target",
|
|
"fortran",
|
|
"sampling_duration",
|
|
"no_tmp_files",
|
|
"rccl",
|
|
"rocshmem",
|
|
"roctx",
|
|
"time_window",
|
|
"transpose",
|
|
"nic",
|
|
"ainic",
|
|
"network",
|
|
"fork",
|
|
"user_api",
|
|
"thread_limit",
|
|
"pthreads",
|
|
"rewrite_caller",
|
|
"locks",
|
|
"caller_include",
|
|
"causal",
|
|
"causal_e2e",
|
|
"papi",
|
|
"code_coverage",
|
|
"lulesh",
|
|
"unit_tests",
|
|
"hip_stream",
|
|
"presets",
|
|
"cli_help",
|
|
"hpc",
|
|
"hip",
|
|
"scratch_memory",
|
|
"rocm",
|
|
"kfd",
|
|
"unified_memory",
|
|
"validation_usm",
|
|
"selective_regions",
|
|
"minimal",
|
|
"rank_filter",
|
|
"pytest_impl",
|
|
]
|
|
for label in non_functional_markers + generic_functional_markers:
|
|
config.addinivalue_line("markers", f"{label}: label test as {label}")
|
|
|
|
# Keep a module-level ref for hooks that don't receive config directly
|
|
pytest._config_ref = config
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Collection hooks
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def pytest_generate_tests(metafunc):
|
|
"""Dynamically parametrize tests based on markers."""
|
|
marker = metafunc.definition.get_closest_marker("python_versions")
|
|
if marker is not None:
|
|
rocprof_config = get_rocprof_config()
|
|
supported = set(rocprof_config.capabilities.supported_python_versions or [])
|
|
|
|
# When --python-versions is explicitly passed, always parametrize
|
|
# with those exact versions so node IDs match.
|
|
# Unsupported versions are marked as "skip"
|
|
pytest_config = getattr(pytest, "_config_ref", None)
|
|
requested_str = (
|
|
pytest_config.getoption("--python-versions", default=None)
|
|
if pytest_config
|
|
else None
|
|
)
|
|
if requested_str:
|
|
requested = [v.strip() for v in requested_str.split(";") if v.strip()]
|
|
params = []
|
|
for ver in requested:
|
|
if ver in supported:
|
|
params.append(ver)
|
|
else:
|
|
params.append(
|
|
pytest.param(
|
|
ver,
|
|
marks=pytest.mark.skip(f"Python {ver} not available"),
|
|
)
|
|
)
|
|
metafunc.parametrize("python_version", params)
|
|
elif supported:
|
|
metafunc.parametrize("python_version", sorted(supported))
|
|
else:
|
|
metafunc.parametrize(
|
|
"python_version",
|
|
[pytest.param(None, marks=pytest.mark.skip("No Python versions found"))],
|
|
)
|
|
|
|
|
|
# Used by the run_if_gpu_category marker
|
|
def gpu_category_eval_context() -> dict[str, bool]:
|
|
info = get_gpu_info()
|
|
return {
|
|
"instinct": info is not None and "instinct" in info.categories,
|
|
"radeon": info is not None and "radeon" in info.categories,
|
|
"apu": info is not None and "apu" in info.categories,
|
|
}
|
|
|
|
|
|
def pytest_collection_modifyitems(config, items) -> None:
|
|
"""Modify items based on markers."""
|
|
verbose = config.option.verbose > 0
|
|
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
except Exception as e:
|
|
pytest.exit(f"{e}")
|
|
|
|
# ----------------------------------------------------------------------------
|
|
def base_modifications(item: pytest.Item) -> None:
|
|
"""This function should be called for every item."""
|
|
_standardize_test_name(item, config, verbose=verbose)
|
|
|
|
# Handle optional markers
|
|
# The general form is <name>_optional(...). If the condition is met, <name> marker is added
|
|
if (
|
|
"mpi_optional" in item.keywords
|
|
and mpi_unavailable_reason(rocprof_config, config) is None
|
|
):
|
|
target = item.get_closest_marker("mpi_optional").args[0]
|
|
try:
|
|
target_path = rocprof_config.get_target_executable(target)
|
|
if rocprof_config.capabilities.target_support_mpi(target_path):
|
|
item.add_marker(pytest.mark.mpi)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
# Marker dependencies
|
|
add_marker_if(
|
|
item,
|
|
"papi",
|
|
req_mark="annotate",
|
|
unavailable_reason=lambda: annotate_unavailable_reason(rocprof_config),
|
|
)
|
|
add_marker_if(item, "mpi", req_mark="mpi_implementation")
|
|
add_marker_if(item, "python", req_mark="python_versions")
|
|
add_marker_if(item, "gpu", req_mark="multi_gpu")
|
|
|
|
# Add corresponding runner type markers based on parametrized values ("mode")
|
|
detected_runners: set[str] = set()
|
|
if hasattr(item, "callspec") and item.callspec:
|
|
params = item.callspec.params
|
|
for param_name in ["runner", "mode", "instrumentation_mode"]:
|
|
if param_name in params:
|
|
value = str(params[param_name])
|
|
if value in ROCPROFSYS_RUNNER_NAMES:
|
|
detected_runners.add(value)
|
|
for runner in detected_runners:
|
|
marker_name = runner.replace("-", "_")
|
|
item.add_marker(getattr(pytest.mark, marker_name))
|
|
|
|
# ----------------------------------------------------------------------------
|
|
|
|
# We will not be running tests in this mode, so marker checks are redundant
|
|
if config.getoption("--ctest-mode", default="off") == "generate":
|
|
for item in items:
|
|
base_modifications(item)
|
|
return
|
|
|
|
# Marker checks
|
|
# "Skip" markers are left for runtime evaluation
|
|
for item in items:
|
|
base_modifications(item)
|
|
if "build_only" in item.keywords and rocprof_config.is_installed:
|
|
item.add_marker(
|
|
pytest.mark.skip(reason="Test only runs in build mode (build_only)")
|
|
)
|
|
if "gpu" in item.keywords:
|
|
_msg = gpu_unavailable_reason()
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "ucx" in item.keywords and not rocprof_config.capabilities.ucx_availability:
|
|
item.add_marker(pytest.mark.skip(reason="UCX not available"))
|
|
if "mpi" in item.keywords:
|
|
_msg = mpi_unavailable_reason(rocprof_config, config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "mpi_implementation" in item.keywords:
|
|
req_impl = item.get_closest_marker("mpi_implementation").args[0]
|
|
if req_impl != rocprof_config.capabilities.mpi_implementation:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"Requires {req_impl}, but {rocprof_config.capabilities.mpi_implementation} found"
|
|
)
|
|
)
|
|
if "overflow" in item.keywords:
|
|
_msg = overflow_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "attach" in item.keywords:
|
|
_msg = attach_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "python" in item.keywords:
|
|
_msg = python_base_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
_msg = python_versions_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "julia" in item.keywords:
|
|
_msg = julia_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "xnack" in item.keywords:
|
|
_msg = xnack_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "no_docker" in item.keywords:
|
|
_msg = no_docker_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "shmem" in item.keywords:
|
|
_msg = shmem_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "nic" in item.keywords:
|
|
_msg = nic_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "ainic_required" in item.keywords:
|
|
_msg = ainic_unavailable_reason(rocprof_config)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "rocprofiler_sdk_min_version" in item.keywords:
|
|
req_version = item.get_closest_marker("rocprofiler_sdk_min_version").args[0]
|
|
_msg = rocprofiler_sdk_min_version_unavailable_reason(
|
|
rocprof_config, req_version
|
|
)
|
|
if _msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_msg))
|
|
if "rocm_min_version" in item.keywords:
|
|
req_version = item.get_closest_marker("rocm_min_version").args[0]
|
|
system_version = rocprof_config.rocm_version
|
|
if system_version is None:
|
|
item.add_marker(pytest.mark.skip(reason="ROCm version not found"))
|
|
# Parse min_version and compare
|
|
min_parts = req_version.split(".")
|
|
min_tuple = tuple(int(p) for p in (min_parts + ["0", "0"])[:3])
|
|
if system_version < min_tuple:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"ROCm {'.'.join(map(str, system_version))} < required {req_version}"
|
|
)
|
|
)
|
|
if "oshrun_min_version" in item.keywords:
|
|
req_version = item.get_closest_marker("oshrun_min_version").args[0]
|
|
system_version = rocprof_config.capabilities.oshrun_version
|
|
if system_version is None:
|
|
item.add_marker(pytest.mark.skip(reason="OpenSHMEM version not found"))
|
|
else:
|
|
min_parts = req_version.split(".")
|
|
min_tuple = tuple(int(p) for p in (min_parts + ["0", "0"])[:2])
|
|
if system_version < min_tuple:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"oshrun version {'.'.join(map(str, system_version))} < required {req_version}"
|
|
)
|
|
)
|
|
if "amdsmi_min_version" in item.keywords:
|
|
req_version = item.get_closest_marker("amdsmi_min_version").args[0]
|
|
system_version = rocprof_config.capabilities.amdsmi_version
|
|
if system_version is None:
|
|
item.add_marker(pytest.mark.skip(reason="AMD-SMI version not found"))
|
|
else:
|
|
min_parts = req_version.split(".")
|
|
min_tuple = tuple(int(p) for p in (min_parts + ["0", "0"])[:2])
|
|
if system_version < min_tuple:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"AMD-SMI {'.'.join(map(str, system_version))} < required {req_version}"
|
|
)
|
|
)
|
|
if "amdgpu_min_version" in item.keywords:
|
|
req_version = item.get_closest_marker("amdgpu_min_version").args[0]
|
|
system_version = rocprof_config.capabilities.amdgpu_version
|
|
if system_version is None:
|
|
item.add_marker(pytest.mark.skip(reason="amdgpu version not found"))
|
|
else:
|
|
min_parts = req_version.split(".")
|
|
min_tuple = tuple(int(p) for p in (min_parts + ["0", "0", "0"])[:3])
|
|
if system_version < min_tuple:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"amdgpu {'.'.join(map(str, system_version))} < required {req_version}"
|
|
)
|
|
)
|
|
if "run_if_gpu_category" in item.keywords:
|
|
_gpu_msg = gpu_unavailable_reason()
|
|
if _gpu_msg is not None:
|
|
item.add_marker(pytest.mark.skip(reason=_gpu_msg))
|
|
expr = item.get_closest_marker("run_if_gpu_category").args[0]
|
|
try:
|
|
result = eval(expr, {"__builtins__": {}}, gpu_category_eval_context())
|
|
if not result:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"GPU category condition '{expr}' is False"
|
|
)
|
|
)
|
|
except Exception as e:
|
|
pytest.exit(f"Invalid run_if_gpu_category expression: {e}", returncode=1)
|
|
if "multi_gpu" in item.keywords:
|
|
num_gpu = item.get_closest_marker("multi_gpu").args[0]
|
|
info = get_gpu_info()
|
|
if info.device_count < num_gpu:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=f"Test requires atleast {num_gpu} GPUs but system has {info.device_count}"
|
|
)
|
|
)
|
|
|
|
|
|
def pytest_collection_finish(session):
|
|
if session.config.getoption("--ctest-mode", default="off") == "generate":
|
|
raw_path = session.config.getoption("--ctest-output-path", default=None)
|
|
output_path = Path(raw_path) if raw_path else None
|
|
_ctest_generate_tests(session.items, output_path)
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Test execution hooks
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.hookimpl(hookwrapper=True) # Allows yield
|
|
def pytest_runtest_makereport(item, call):
|
|
"""
|
|
Attaches a "Runner Output" section to the call-phase report of the form:
|
|
|
|
=========================================
|
|
Command: <command>
|
|
Environment:
|
|
<environment>
|
|
=========================================
|
|
Test Output:
|
|
<test output>
|
|
=========================================
|
|
"""
|
|
outcome = yield
|
|
rep = outcome.get_result()
|
|
|
|
setattr(item, f"rep_{rep.when}", rep)
|
|
|
|
if rep.when != "call" or item.stash.get(_output_printed_key, False):
|
|
return
|
|
|
|
item.stash[_output_printed_key] = True
|
|
test_result = item.stash.get(_result_key, None)
|
|
if test_result is None:
|
|
return
|
|
|
|
report_output = []
|
|
cmd = " ".join(str(c) for c in getattr(test_result, "command", []))
|
|
if cmd:
|
|
report_output.append(f"{'='*70}")
|
|
report_output.append(f"Command: {cmd}")
|
|
test_env = getattr(test_result, "environment", None)
|
|
if isinstance(test_env, environment.TestEnvironment):
|
|
env_lines = test_env.format_layers()
|
|
if env_lines:
|
|
report_output.append("Environment:\n\n" + "\n".join(env_lines) + "\n")
|
|
report_output.append(f"{'='*70}")
|
|
test_output = getattr(test_result, "test_output", "")
|
|
extra_output = getattr(test_result, "extra_output", None)
|
|
if test_output or extra_output:
|
|
report_output.append("Test Output:\n")
|
|
if test_output:
|
|
report_output.append(test_output)
|
|
if extra_output:
|
|
report_output.append(extra_output)
|
|
report_output.append(f"{'='*70}")
|
|
|
|
if not report_output:
|
|
return
|
|
|
|
rep.sections.append(("Runner Output", "\n".join(report_output) + "\n\n"))
|
|
|
|
|
|
def pytest_runtest_logreport(report):
|
|
"""Print the runner output inline for passing tests in CTest run mode.
|
|
|
|
Failing tests already have their "Runner Output" section printed by pytest's
|
|
failure summary.
|
|
"""
|
|
config = getattr(pytest, "_config_ref", None)
|
|
if config is None:
|
|
return
|
|
if config.getoption("--ctest-mode", default="off") != "run":
|
|
return
|
|
if report.when != "call" or not report.passed:
|
|
return
|
|
|
|
terminal = config.pluginmanager.get_plugin("terminalreporter")
|
|
if terminal is None:
|
|
return
|
|
|
|
for section_name, section_content in report.sections:
|
|
if section_name == "Runner Output":
|
|
terminal.write_line(f"\n--- {section_name} ---")
|
|
for line in section_content.splitlines():
|
|
terminal.write_line(line)
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Session End hooks
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def pytest_sessionfinish(session, exitstatus):
|
|
"""Code that runs after tests complete
|
|
|
|
In CTest mode, map "all skipped" to exit code SKIP_RETURN_CODE
|
|
so that CTest can distinguish skipped from passed (via SKIP_RETURN_CODE).
|
|
"""
|
|
if (
|
|
session.config.getoption("--ctest-mode", default="off") == "run"
|
|
and exitstatus == 0
|
|
):
|
|
reporter = session.config.pluginmanager.get_plugin("terminalreporter")
|
|
if reporter is not None:
|
|
passed = len(reporter.stats.get("passed", []))
|
|
skipped = len(reporter.stats.get("skipped", []))
|
|
if passed == 0 and skipped > 0:
|
|
session.exitstatus = SKIP_RETURN_CODE
|
|
|
|
|
|
# ============================================================================
|
|
#
|
|
# Helper functions
|
|
#
|
|
# ============================================================================
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Collection-time availability: return None if OK, else a skip reason string.
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def overflow_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if rocprof_config.capabilities.perf_events_usable:
|
|
return None
|
|
return "Requires either perf_event_paranoid <= 2 or CAP_SYS_ADMIN to be available"
|
|
|
|
|
|
def gpu_unavailable_reason() -> Optional[str]:
|
|
gpu_info = get_gpu_info()
|
|
if gpu_info is not None and gpu_info.available:
|
|
return None
|
|
return "No valid GPU available"
|
|
|
|
|
|
def annotate_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
msg = overflow_unavailable_reason(rocprof_config)
|
|
if msg is not None:
|
|
return msg
|
|
if not rocprof_config.capabilities.papi_availability:
|
|
return "PAPI not available"
|
|
return None
|
|
|
|
|
|
def attach_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if rocprof_config.capabilities.ptrace_scope == 0:
|
|
return None
|
|
return (
|
|
"Requires ptrace_scope to be 0. Run 'echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope' "
|
|
"to enable attaching to process"
|
|
)
|
|
|
|
|
|
def nic_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
caps = rocprof_config.capabilities
|
|
if caps.papi_nic_events is not None and caps.perf_events_usable:
|
|
return None
|
|
return "Requires PAPI network events and perf_event_paranoid <= 2 (or CAP_SYS_ADMIN) to be available"
|
|
|
|
|
|
def ainic_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
"""Check if AI NIC tracking is available.
|
|
|
|
Requires ``amd-smi static`` to report at least one NETDEV entry.
|
|
"""
|
|
if not rocprof_config.capabilities.ai_nic_devices:
|
|
return "No AI NIC devices found (amd-smi static reports no NETDEV entries)"
|
|
return None
|
|
|
|
|
|
def rocprofiler_sdk_min_version_unavailable_reason(
|
|
rocprof_config: RocprofsysConfig, req_version: str
|
|
) -> Optional[str]:
|
|
"""Return a skip reason if the detected rocprofiler-sdk is older than req_version.
|
|
|
|
``req_version`` is a dotted version string (e.g. ``"1.2.2"``) supplied by the
|
|
test via the ``rocprofiler_sdk_min_version`` marker.
|
|
"""
|
|
system_version = rocprof_config.capabilities.rocprofiler_sdk_version
|
|
min_parts = req_version.split(".")
|
|
min_tuple = tuple(int(p) for p in (min_parts + ["0", "0", "0"])[:3])
|
|
if system_version is not None and system_version >= min_tuple:
|
|
return None
|
|
_found = (
|
|
".".join(map(str, system_version)) if system_version is not None else "not found"
|
|
)
|
|
return (
|
|
f"Requires rocprofiler-sdk minimum {req_version}, "
|
|
f"but system detected version {_found}"
|
|
)
|
|
|
|
|
|
def mpi_unavailable_reason(
|
|
rocprof_config: RocprofsysConfig, config: pytest.Config
|
|
) -> Optional[str]:
|
|
if rocprof_config.capabilities.mpiexec_exec is None:
|
|
return "MPI not available"
|
|
return None
|
|
|
|
|
|
def python_base_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if rocprof_config.rocprofsys_python is not None:
|
|
return None
|
|
return "rocprof-sys-python binary not found"
|
|
|
|
|
|
def python_versions_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if (
|
|
rocprof_config.capabilities.supported_python_versions is not None
|
|
and os.environ.get("ROCPROFSYS_USE_PYTHON", "ON").upper() == "ON"
|
|
):
|
|
return None
|
|
return (
|
|
"No supported Python versions. Each version needs a corresponding "
|
|
"libpyrocprofsys.<IMPL>-<VERSION>-<ARCH>-<OS>-<ABI>.so in site-packages/rocprofsys."
|
|
)
|
|
|
|
|
|
def julia_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if rocprof_config.capabilities.julia_exec:
|
|
return None
|
|
return "Julia not available"
|
|
|
|
|
|
def xnack_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if get_xnack_support(rocprof_config.rocm_path):
|
|
return None
|
|
return "XNACK not supported"
|
|
|
|
|
|
def no_docker_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if not rocprof_config.capabilities.is_inside_docker:
|
|
return None
|
|
return "Test cannot run inside a Docker container"
|
|
|
|
|
|
def shmem_unavailable_reason(rocprof_config: RocprofsysConfig) -> Optional[str]:
|
|
if rocprof_config.capabilities.oshrun_exec:
|
|
return None
|
|
return "SHMEM not available"
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Test-category (tier) label injection from test_categories.yaml
|
|
# ----------------------------------------------------------------------------
|
|
# Single source of truth for tier policy is tests/test_categories.yaml.
|
|
# At CTest-generate time we read the YAML and append tier labels to each
|
|
# test's emitted LABELS set, so `ctest -L <tier>` Just Works from the
|
|
# installed share/rocprofiler-systems/tests directory.
|
|
|
|
TIER_ORDER = ["quick", "standard", "comprehensive", "full"]
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _load_test_categories() -> Optional[dict]:
|
|
"""Load and compile test_categories.yaml from rocprofsys_tests_dir.
|
|
|
|
Reads the YAML that CMake installs/configures into
|
|
``<build|install>/share/rocprofiler-systems/tests``
|
|
|
|
Returns ``None`` (with a single STDERR warning) when the YAML is missing or
|
|
PyYAML isn't importable
|
|
"""
|
|
if yaml is None:
|
|
print(
|
|
"[test_categories] PyYAML not available - skipping tier label injection.",
|
|
file=sys.stderr,
|
|
)
|
|
return None
|
|
try:
|
|
yaml_path = get_rocprof_config().rocprofsys_tests_dir / "test_categories.yaml"
|
|
except Exception as exc:
|
|
print(
|
|
f"[test_categories] Could not resolve tests dir - skipping tier label injection: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
return None
|
|
if not yaml_path.exists():
|
|
print(
|
|
f"[test_categories] {yaml_path} not found - skipping tier label injection.",
|
|
file=sys.stderr,
|
|
)
|
|
return None
|
|
try:
|
|
data = yaml.safe_load(yaml_path.read_text()) or {}
|
|
except yaml.YAMLError as exc:
|
|
print(
|
|
f"[test_categories] Failed to load {yaml_path}: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
return None
|
|
|
|
def _compile_list(patterns):
|
|
compiled = []
|
|
for p in patterns or []:
|
|
# Flatten one level: a YAML alias item (e.g. `- *common_excludes`)
|
|
# expands to a nested list, so callers can mix a shared anchor with
|
|
# per-tier additions.
|
|
for pattern in p if isinstance(p, list) else [p]:
|
|
try:
|
|
compiled.append(re.compile(pattern))
|
|
except re.error as exc:
|
|
print(
|
|
f"[test_categories] Skipping invalid regex {pattern!r}: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
return compiled
|
|
|
|
def _flatten_labels(values):
|
|
# One-level flatten (like _compile_list) so a YAML alias item expands
|
|
# to a nested list without raising TypeError on the unhashable inner
|
|
# list when set()-ed.
|
|
flat = []
|
|
for v in values or []:
|
|
flat.extend(v if isinstance(v, list) else [v])
|
|
return flat
|
|
|
|
tier_cfg: dict = {}
|
|
for tier in TIER_ORDER:
|
|
cfg = (data.get("test_categories", {}) or {}).get(tier) or {}
|
|
tier_cfg[tier] = {
|
|
"include": _compile_list(cfg.get("regex_includes")),
|
|
"exclude": _compile_list(cfg.get("regex_excludes")),
|
|
"label_excludes": _compile_list(cfg.get("label_excludes")),
|
|
"label_includes": _compile_list(cfg.get("label_includes")),
|
|
"labels": _flatten_labels(cfg.get("added_supplementary_labels")),
|
|
}
|
|
|
|
return {"tiers": tier_cfg}
|
|
|
|
|
|
def _resolve_tier_labels(test_name: str, existing_labels: set[str]) -> set[str]:
|
|
"""Return tier labels (subset of TIER_ORDER) for *test_name*.
|
|
|
|
Each tier is evaluated independently with the *exact* CTest filter model.
|
|
The four YAML axes map to CTest options as (labels are pytest MARKERs):
|
|
* ``regex_includes`` -> ``-R`` * ``regex_excludes`` -> ``-E``
|
|
* ``label_includes`` -> ``-L`` * ``label_excludes`` -> ``-LE``
|
|
|
|
The rocJenkins-style cascade ("matching quick also yields standard /
|
|
comprehensive / full") is achieved by having those higher tiers use
|
|
broad include patterns (typically ``regex_includes: [".*"]``). Per-tier
|
|
``regex_excludes`` punches a hole through the cascade for individual
|
|
tests: listing ``testA`` under ``standard.regex_excludes`` drops
|
|
``standard`` from its label set even if ``quick`` / ``comprehensive`` /
|
|
``full`` match.
|
|
|
|
In addition to the tier name, each matched tier contributes its
|
|
``added_supplementary_labels:`` to the test's labels.
|
|
"""
|
|
categories = _load_test_categories()
|
|
if not categories:
|
|
return set()
|
|
matched_indices: list[int] = []
|
|
extra_labels: set[str] = set()
|
|
for i, tier in enumerate(TIER_ORDER):
|
|
cfg = categories["tiers"].get(tier) or {}
|
|
include_regex = cfg.get("include", [])
|
|
include_labels = cfg.get("label_includes", [])
|
|
exclude_regex = cfg.get("exclude", [])
|
|
exclude_labels = cfg.get("label_excludes", [])
|
|
# -R: an empty include is a pass-through; otherwise the NAME must match.
|
|
if include_regex and not any(p.search(test_name) for p in include_regex):
|
|
continue
|
|
# -L: an empty include is a pass-through; otherwise a marker label must
|
|
# match. AND-combined with the -R axis above, exactly like CTest.
|
|
if include_labels and not any(
|
|
p.search(label) for p in include_labels for label in existing_labels
|
|
):
|
|
continue
|
|
# -E: NAME matching any exclude pattern vetoes the test.
|
|
if any(p.search(test_name) for p in exclude_regex):
|
|
continue
|
|
# -LE: any marker label matching an exclude pattern vetoes the test.
|
|
if any(p.search(label) for p in exclude_labels for label in existing_labels):
|
|
continue
|
|
matched_indices.append(i)
|
|
extra_labels.update(cfg.get("labels", []))
|
|
return {TIER_ORDER[i] for i in matched_indices} | extra_labels
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# CTest generator functions
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def _cmake_escape(s: str) -> str:
|
|
"""Escape a string for use inside CMake double-quoted arguments."""
|
|
return s.replace("\\", "\\\\").replace('"', '\\"')
|
|
|
|
|
|
def _ctest_item_ctest_identity(item: pytest.Item) -> tuple[str, str, str]:
|
|
"""Return ``(original_nodeid, item name, CTest nodeid fragment)`` for CMake generation."""
|
|
test_id = item.stash.get(_original_nodeid_key, item.nodeid)
|
|
test_name = item.name
|
|
if "::" in test_id:
|
|
file_part, _, rest = test_id.partition("::")
|
|
test_nodeid = f"{Path(file_part).name}::{rest}"
|
|
else:
|
|
test_nodeid = Path(test_id).name
|
|
return test_id, test_name, test_nodeid
|
|
|
|
|
|
def _emit_ctest_header_block() -> list[str]:
|
|
"""CMake preamble for generated CTestTestfile.cmake (env, paths, pytest/python discovery)."""
|
|
return [
|
|
"# Auto-generated CTest definitions from rocprofiler-systems pytest suite",
|
|
"# DO NOT EDIT — regenerate via: pytest <dir> --ctest-mode=generate",
|
|
"#",
|
|
"# Run with: ctest --test-dir <directory containing this file>",
|
|
"#",
|
|
"# Supported environment variables:",
|
|
"# ROCPROFSYS_TEST_DIR - Path to test package directory or .pyz file",
|
|
"# ROCPROFSYS_TEST_EXECUTABLE - Python or pytest executable to use",
|
|
"# ROCPROFSYS_PYTHON_HINTS - Additional search paths for versioned Python interpreters",
|
|
"#",
|
|
"",
|
|
"if(DEFINED ENV{ROCPROFSYS_TEST_DIR} AND NOT DEFINED ROCPROFSYS_TEST_DIR)",
|
|
' set(ROCPROFSYS_TEST_DIR "$ENV{ROCPROFSYS_TEST_DIR}")',
|
|
"endif()",
|
|
"if(DEFINED ENV{ROCPROFSYS_TEST_EXECUTABLE} AND NOT DEFINED ROCPROFSYS_TEST_EXECUTABLE)",
|
|
' set(ROCPROFSYS_TEST_EXECUTABLE "$ENV{ROCPROFSYS_TEST_EXECUTABLE}")',
|
|
"endif()",
|
|
"if(DEFINED ENV{ROCPROFSYS_PYTHON_HINTS} AND NOT DEFINED ROCPROFSYS_PYTHON_HINTS)",
|
|
' set(ROCPROFSYS_PYTHON_HINTS "$ENV{ROCPROFSYS_PYTHON_HINTS}")',
|
|
"endif()",
|
|
"",
|
|
"execute_process(COMMAND pwd OUTPUT_VARIABLE _CTEST_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)",
|
|
"if(NOT DEFINED ROCPROFSYS_TEST_DIR)",
|
|
' set(ROCPROFSYS_TEST_DIR "${_CTEST_DIR}")',
|
|
"endif()",
|
|
"",
|
|
'set(_INSTALL_PATH "${ROCPROFSYS_TEST_DIR}/rocprofsys-tests.pyz")',
|
|
'set(_BUILD_PATH "${ROCPROFSYS_TEST_DIR}/../share/rocprofiler-systems/tests/pytest/")',
|
|
'set(_TEST_ARGS "-s" "--ctest-mode" "run")',
|
|
"",
|
|
'if(EXISTS "${_INSTALL_PATH}")',
|
|
" if(NOT DEFINED ROCPROFSYS_TEST_EXECUTABLE)",
|
|
" find_program(ROCPROFSYS_TEST_EXECUTABLE NAMES python3 python HINTS ${ROCPROFSYS_PYTHON_HINTS})",
|
|
" endif()",
|
|
" if(NOT ROCPROFSYS_TEST_EXECUTABLE)",
|
|
" message(FATAL_ERROR",
|
|
' "python executable not found. "',
|
|
' "Set ROCPROFSYS_TEST_EXECUTABLE to the correct path "',
|
|
' "or provide ROCPROFSYS_PYTHON_HINTS to search for the executable.")',
|
|
" endif()",
|
|
' set(_ROCPROFSYS_EXE "${ROCPROFSYS_TEST_EXECUTABLE}")',
|
|
' set(_ROCPROFSYS_EXE_ARGS "${_INSTALL_PATH}")',
|
|
' set(_ROCPROFSYS_NODEID_PFX "")',
|
|
' set(_ROCPROFSYS_EXTRA_ARGS "${_TEST_ARGS}")',
|
|
'elseif(EXISTS "${_BUILD_PATH}")',
|
|
" if(NOT DEFINED ROCPROFSYS_TEST_EXECUTABLE)",
|
|
" find_program(ROCPROFSYS_TEST_EXECUTABLE NAMES pytest pytest3 HINTS ${ROCPROFSYS_PYTHON_HINTS})",
|
|
" endif()",
|
|
" if(NOT ROCPROFSYS_TEST_EXECUTABLE)",
|
|
" message(FATAL_ERROR",
|
|
' "pytest executable not found. "',
|
|
' "Set ROCPROFSYS_TEST_EXECUTABLE to the correct path "',
|
|
' "or provide ROCPROFSYS_PYTHON_HINTS to search for the executable.")',
|
|
" endif()",
|
|
' set(_ROCPROFSYS_EXE "${ROCPROFSYS_TEST_EXECUTABLE}")',
|
|
' set(_ROCPROFSYS_EXE_ARGS "")',
|
|
' set(_ROCPROFSYS_NODEID_PFX "${_BUILD_PATH}")',
|
|
' set(_ROCPROFSYS_EXTRA_ARGS "${_TEST_ARGS}")',
|
|
"else()",
|
|
' message(FATAL_ERROR "Cannot find test package. Set ROCPROFSYS_TEST_DIR=/path/to/rocprofsys-tests.pyz")',
|
|
"endif()",
|
|
"",
|
|
"if(DEFINED ENV{ROCPROFSYS_CI_TIMEOUT})",
|
|
' set(_ROCPROFSYS_CI_TIMEOUT "$ENV{ROCPROFSYS_CI_TIMEOUT}")',
|
|
"endif()",
|
|
"",
|
|
]
|
|
|
|
|
|
def _emit_prerequisite_block() -> list[str]:
|
|
"""``rocprofiler-systems-pytest-config`` prerequisite test (global tmp fixture setup)."""
|
|
return [
|
|
'add_test("rocprofiler-systems-pytest-config" "${_ROCPROFSYS_EXE}"'
|
|
' "${_ROCPROFSYS_EXE_ARGS}"'
|
|
' "${_ROCPROFSYS_NODEID_PFX}" "--show-config-only")',
|
|
'set_tests_properties("rocprofiler-systems-pytest-config" PROPERTIES',
|
|
' FIXTURES_SETUP "rocprofsys-global-tmp-files"',
|
|
' LABELS "prerequisite;global"',
|
|
f" TIMEOUT {CONFIG_TIMEOUT + CONFIG_TIMEOUT_BUFFER}",
|
|
")",
|
|
"",
|
|
]
|
|
|
|
|
|
def _emit_cleanup_block() -> list[str]:
|
|
"""``rocprofiler-systems-test-cleanup`` (global tmp fixture cleanup)."""
|
|
return [
|
|
'add_test("rocprofiler-systems-test-cleanup" "${_ROCPROFSYS_EXE}"'
|
|
' "${_ROCPROFSYS_EXE_ARGS}"'
|
|
' "${_ROCPROFSYS_NODEID_PFX}" "--ctest-mode" "cleanup")',
|
|
'set_tests_properties("rocprofiler-systems-test-cleanup" PROPERTIES',
|
|
' FIXTURES_CLEANUP "rocprofsys-global-tmp-files"',
|
|
' LABELS "cleanup;global"',
|
|
" TIMEOUT 30",
|
|
")",
|
|
"",
|
|
]
|
|
|
|
|
|
def _emit_test_timeout_block(
|
|
item: pytest.Item, timeout_buffer: int = CTEST_TIMEOUT_BUFFER
|
|
) -> list[str]:
|
|
"""One CMake block: set ``_TEST_TIMEOUT`` from ``ROCPROFSYS_CI_TIMEOUT`` or pytest timeout (+ buffer)."""
|
|
timeout_marker = item.get_closest_marker("timeout")
|
|
timeout = (
|
|
int(timeout_marker.args[0])
|
|
if timeout_marker and timeout_marker.args
|
|
else DEFAULT_TIMEOUT
|
|
)
|
|
default_timeout = timeout + timeout_buffer
|
|
return [
|
|
"if(DEFINED _ROCPROFSYS_CI_TIMEOUT)",
|
|
f' math(EXPR _TEST_TIMEOUT "${{_ROCPROFSYS_CI_TIMEOUT}} + {timeout_buffer}")',
|
|
"else()",
|
|
f" set(_TEST_TIMEOUT {default_timeout})",
|
|
"endif()",
|
|
"",
|
|
]
|
|
|
|
|
|
def _emit_test_item_block(
|
|
item: pytest.Item,
|
|
labels: set[str],
|
|
depends_on: list[str],
|
|
run_serial: bool,
|
|
) -> list[str]:
|
|
"""``add_test`` + ``set_tests_properties`` for one item (timeout block emitted separately)."""
|
|
_, test_name, test_nodeid = _ctest_item_ctest_identity(item)
|
|
escaped_name = _cmake_escape(test_name)
|
|
escaped_nodeid = _cmake_escape(test_nodeid)
|
|
|
|
# Check if the test runs on a specific python version
|
|
extra_args = ""
|
|
if hasattr(item, "callspec") and "python_version" in item.callspec.params:
|
|
py_ver = item.callspec.params["python_version"]
|
|
if py_ver is not None:
|
|
extra_args += f' "--python-versions={py_ver}"'
|
|
|
|
lines_out: list[str] = [
|
|
f'add_test("{escaped_name}" "${{_ROCPROFSYS_EXE}}"'
|
|
f' "${{_ROCPROFSYS_EXE_ARGS}}"'
|
|
f' "${{_ROCPROFSYS_NODEID_PFX}}{escaped_nodeid}"'
|
|
f"{extra_args} ${{_ROCPROFSYS_EXTRA_ARGS}})"
|
|
]
|
|
props: list[str] = []
|
|
if labels:
|
|
props.append(f' LABELS "{";".join(sorted(labels))}"')
|
|
props.append(" TIMEOUT ${_TEST_TIMEOUT}")
|
|
props.append(f" SKIP_RETURN_CODE {SKIP_RETURN_CODE}")
|
|
props.append(' FIXTURES_REQUIRED "rocprofsys-global-tmp-files"')
|
|
if run_serial:
|
|
props.append(" RUN_SERIAL TRUE")
|
|
if depends_on:
|
|
deps_str = ";".join(_cmake_escape(d) for d in depends_on)
|
|
props.append(f' DEPENDS "{deps_str}"')
|
|
|
|
lines_out.append(f'set_tests_properties("{escaped_name}" PROPERTIES')
|
|
lines_out.extend(props)
|
|
lines_out.append(")")
|
|
lines_out.append("")
|
|
return lines_out
|
|
|
|
|
|
def _ctest_generate_tests(
|
|
items: list[pytest.Item], output_path: Optional[Path] = None
|
|
) -> None:
|
|
"""Generate a CTestTestfile.cmake file and print it to stdout."""
|
|
|
|
no_report_markers = {
|
|
"parametrize", # Ignored, except for "mode" parameter (instrumentation mode)
|
|
# Pytest built-in
|
|
"usefixtures",
|
|
"filterwarnings",
|
|
"skipif",
|
|
"skip",
|
|
"xfail",
|
|
# Internal markers
|
|
"python_versions",
|
|
"mpi_optional",
|
|
"no_docker",
|
|
"oshrun_min_version",
|
|
"rocm_min_version",
|
|
"amdsmi_min_version",
|
|
"amdgpu_min_version",
|
|
"rocprofiler_sdk_min_version",
|
|
"run_if_gpu_category",
|
|
"preserve",
|
|
# For CTests
|
|
"timeout",
|
|
"depends_on",
|
|
"serialize",
|
|
"class_name",
|
|
}
|
|
no_report_args_markers = {"rocpd"}
|
|
only_report_args_markers = {"mpi_implementation"}
|
|
|
|
lines = _emit_ctest_header_block()
|
|
lines.extend(_emit_prerequisite_block())
|
|
|
|
seen_names: dict[str, str] = {} # escaped_name -> original nodeid
|
|
|
|
for item in items:
|
|
test_id, test_name, _ = _ctest_item_ctest_identity(item)
|
|
|
|
# Handle certain markers that affect how CTest is configured
|
|
|
|
labels: set[str] = set()
|
|
depends_on: list[str] = []
|
|
run_serial = False
|
|
|
|
depends_marker = item.get_closest_marker("depends_on")
|
|
if depends_marker:
|
|
depends_on.extend(str(arg) for arg in depends_marker.args)
|
|
|
|
if item.get_closest_marker("serialize"):
|
|
run_serial = True
|
|
|
|
if hasattr(item, "callspec") and "mode" in item.callspec.params:
|
|
labels.add(str(item.callspec.params["mode"]))
|
|
|
|
# Translate pytest markers to CTest labels
|
|
|
|
for marker in item.iter_markers():
|
|
if marker.name in no_report_markers:
|
|
continue
|
|
if marker.name in only_report_args_markers:
|
|
for arg in marker.args:
|
|
labels.add(str(arg))
|
|
continue
|
|
if marker.name in no_report_args_markers or not marker.args:
|
|
labels.add(marker.name)
|
|
else:
|
|
args_str = ", ".join(str(a) for a in marker.args)
|
|
labels.add(f"{marker.name}[{args_str}]")
|
|
|
|
# Inject tier (quick/standard/comprehensive/full) labels from
|
|
# test_categories.yaml
|
|
labels |= _resolve_tier_labels(test_name, labels)
|
|
|
|
escaped_name = _cmake_escape(test_name)
|
|
|
|
if escaped_name in seen_names:
|
|
pytest.exit(
|
|
f"\nDuplicate CTest name '{escaped_name}' generated from:\n"
|
|
f" 1) {seen_names[escaped_name]}\n"
|
|
f" 2) {test_id}\n"
|
|
f"(Due to _standardize_test_name or parametrization)\n"
|
|
f"Rework test name or parametrization to produce unique names.",
|
|
returncode=1,
|
|
)
|
|
seen_names[escaped_name] = test_id
|
|
|
|
lines.extend(_emit_test_timeout_block(item))
|
|
lines.extend(
|
|
_emit_test_item_block(
|
|
item,
|
|
labels,
|
|
depends_on,
|
|
run_serial,
|
|
)
|
|
)
|
|
|
|
lines.extend(_emit_cleanup_block())
|
|
|
|
content = "\n".join(lines)
|
|
if output_path:
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text(content)
|
|
print(f"Generated {len(items)} CTest definitions -> {output_path}")
|
|
else:
|
|
print(content)
|
|
pytest.exit("CTest generation complete", returncode=0)
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Cache warmup
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_object_cached_properties(obj: object) -> None:
|
|
"""Access every ``persistent_cached_property`` on ``obj`` to compute+store it."""
|
|
from rocprofsys.cache import SerializationError, persistent_cached_property
|
|
|
|
seen: set[str] = set()
|
|
for klass in type(obj).__mro__:
|
|
for name, attr in vars(klass).items():
|
|
if name in seen or not isinstance(attr, persistent_cached_property):
|
|
continue
|
|
seen.add(name)
|
|
try:
|
|
getattr(obj, name)
|
|
except SerializationError:
|
|
raise
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _reset_cache_for_regeneration(store) -> None:
|
|
"""Discard the previous run's cache so every probe recomputes and rewrites.
|
|
|
|
Clears the on-disk store and the in-process memoization on
|
|
``get_rocprof_config`` so nothing stale carries over.
|
|
"""
|
|
cache_existed = store.path.exists()
|
|
print(
|
|
f"Overwriting old cache file: {store.path}"
|
|
if cache_existed
|
|
else f"Generating cache file: {store.path}"
|
|
)
|
|
store.clear()
|
|
try:
|
|
get_rocprof_config.cache_clear()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _warm_gpu_probes(rocprof_config: RocprofsysConfig) -> None:
|
|
"""Compute + store the GPU / ROCm tooling probes for the active ROCm path."""
|
|
from rocprofsys.cache import SerializationError
|
|
|
|
for probe in (detect_gpu, get_offload_extractor, get_xnack_support):
|
|
try:
|
|
probe(rocprof_config.rocm_path)
|
|
except SerializationError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"Warning: seeding {probe.__name__} failed: {e}")
|
|
|
|
|
|
def _warm_cached_properties(rocprof_config: RocprofsysConfig) -> None:
|
|
"""Seed every non-target-dependent persistent_cached_property (config + capabilities)."""
|
|
from rocprofsys.cache import SerializationError
|
|
|
|
_seed_object_cached_properties(rocprof_config)
|
|
try:
|
|
cap = rocprof_config.capabilities
|
|
except SerializationError:
|
|
raise
|
|
except Exception:
|
|
cap = None
|
|
if cap is not None:
|
|
_seed_object_cached_properties(cap)
|
|
|
|
|
|
def _seed_capability_cache() -> None:
|
|
"""(Re)generate the persistent capability cache during the config setup step.
|
|
|
|
This is handled by ``rocprofiler-systems-pytest-config`` test (``--show-config-only``).
|
|
This is always run once via ``FIXTURES_SETUP`` and ``FIXTURE_REQUIRED``.
|
|
"""
|
|
from rocprofsys.cache import SerializationError, get_shared_cache
|
|
|
|
store = get_shared_cache()
|
|
if store is None:
|
|
return # caching disabled/unavailable
|
|
|
|
_reset_cache_for_regeneration(store)
|
|
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
except SerializationError:
|
|
raise
|
|
except Exception:
|
|
return
|
|
|
|
_warm_gpu_probes(rocprof_config)
|
|
_warm_cached_properties(rocprof_config)
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Other helpers
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
def _standardize_test_name(
|
|
item: pytest.Item, config: pytest.Config, verbose: bool = False
|
|
) -> None:
|
|
|
|
# Strip test prefix from the test method name
|
|
test_name = item.name
|
|
if test_name.startswith("test"):
|
|
test_name = test_name[4:]
|
|
if test_name.startswith(("_", "-")):
|
|
test_name = test_name[1:]
|
|
|
|
class_name = None
|
|
name_marker = item.get_closest_marker("class_name")
|
|
if name_marker and name_marker.args:
|
|
class_name = str(name_marker.args[0]).strip()
|
|
|
|
if class_name:
|
|
full_name = f"{class_name}-{test_name}"
|
|
elif item.cls:
|
|
py_class = item.cls.__name__
|
|
if py_class.startswith("Test"):
|
|
py_class = py_class[4:]
|
|
full_name = f"{py_class}-{test_name}"
|
|
else:
|
|
full_name = test_name
|
|
|
|
formatted_name = "".join(c if c.isalnum() or c == "." else "-" for c in full_name)
|
|
formatted_name = formatted_name.replace("_", "-")
|
|
while "--" in formatted_name:
|
|
formatted_name = formatted_name.replace("--", "-")
|
|
formatted_name = formatted_name.strip("-")
|
|
formatted_name = formatted_name.lower()
|
|
|
|
item.stash[_original_nodeid_key] = item.nodeid
|
|
# nodeid is what is used to display the test name in the terminal
|
|
# By default, it groups it by module. In verbose, it shows the full path + class + method
|
|
# To get a cleaner output in verbose mode, we modify the nodeid but only if verbose is True
|
|
# This avoids breaking the default grouping by module in non-verbose mode
|
|
if verbose:
|
|
item._nodeid = formatted_name
|
|
item.name = formatted_name
|
|
|
|
# Allow -k filtering by the formatted name
|
|
item.extra_keyword_matches.add(formatted_name)
|
|
item.extra_keyword_matches.add(formatted_name.lower())
|
|
|
|
|
|
def _config_timeout_handler(signum, frame):
|
|
"""SIGALRM handler for ``--show-config-only``: dump where we hung, then hard-exit."""
|
|
sys.stderr.write(
|
|
"\n" + "=" * 70 + "\n"
|
|
f"[rocprofiler-systems-pytest-config] ERROR: configuration probing exceeded "
|
|
f"{CONFIG_TIMEOUT}s and was aborted.\n"
|
|
"Stack trace at the point of timeout:\n" + "-" * 70 + "\n"
|
|
)
|
|
traceback.print_stack(frame, file=sys.stderr)
|
|
sys.stderr.write("=" * 70 + "\n")
|
|
sys.stderr.flush()
|
|
# os._exit avoids re-entering the (possibly blocked) interpreter state and ensures
|
|
# the process actually dies even if we interrupted an uninterruptible-looking call.
|
|
os._exit(2)
|
|
|
|
|
|
def _generate_rocprofsys_config_header() -> list[str]:
|
|
"""Generate the config header under an internal SIGALRM watchdog.
|
|
|
|
The system probes below (rocminfo, GPU detection, MPI/oshrun version, AMD-SMI/PAPI)
|
|
can block indefinitely. Without a guard a hang would surface as a silent CTest
|
|
``***Timeout``; the watchdog instead prints a diagnostic stack trace and hard-exits.
|
|
A plain exception would be swallowed by a blocking syscall, so the handler forces
|
|
an immediate non-zero exit (see :func:`_config_timeout_handler`).
|
|
"""
|
|
_watchdog_active = hasattr(signal, "SIGALRM") # False on non-POSIX platforms
|
|
_previous_handler = None
|
|
if _watchdog_active:
|
|
try:
|
|
_previous_handler = signal.signal(signal.SIGALRM, _config_timeout_handler)
|
|
signal.alarm(CONFIG_TIMEOUT)
|
|
except ValueError:
|
|
# signal handlers can only be installed from the main thread of the main
|
|
# interpreter; if config probing runs elsewhere, skip the watchdog rather
|
|
# than crash the probe.
|
|
_watchdog_active = False
|
|
|
|
try:
|
|
return _build_rocprofsys_config_header()
|
|
finally:
|
|
if _watchdog_active:
|
|
signal.alarm(0)
|
|
signal.signal(signal.SIGALRM, _previous_handler)
|
|
|
|
|
|
def _build_rocprofsys_config_header() -> list[str]:
|
|
"""Collect system configuration and format it as printable header lines."""
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
cap = rocprof_config.capabilities
|
|
except Exception as e:
|
|
return [f"{e}"]
|
|
|
|
gpu_info = get_gpu_info()
|
|
|
|
# Rocm version
|
|
rocm_version = (
|
|
".".join(map(str, rocprof_config.rocm_version))
|
|
if rocprof_config.rocm_version
|
|
else "Not found"
|
|
)
|
|
|
|
# Rocminfo
|
|
rocminfo_path = get_rocminfo(rocprof_config.rocm_path)
|
|
if not rocminfo_path:
|
|
rocminfo_err_msg = "Not found - Ensure rocminfo is in ROCM_PATH or PATH - Assuming no GPU configuration"
|
|
|
|
# Offload extractor
|
|
offload_msg = None
|
|
tool_path, is_llvm_too_old = get_offload_extractor(rocprof_config.rocm_path)
|
|
if tool_path:
|
|
if tool_path.name == "llvm-objdump":
|
|
offload_msg = f"{tool_path}"
|
|
elif tool_path.name == "roc-obj-ls":
|
|
if not is_llvm_too_old:
|
|
offload_msg = f"Using deprecated {tool_path} - Set ROCM_LLVM_OBJDUMP to use llvm-objdump instead"
|
|
else:
|
|
offload_msg = f"{tool_path}"
|
|
|
|
if not offload_msg:
|
|
offload_msg = (
|
|
"Not found - Set ROCM_LLVM_OBJDUMP to path of llvm-objdump (v20+), "
|
|
"or to path of roc-obj-ls if llvm-objdump < v20"
|
|
)
|
|
xnack_support = get_xnack_support(rocprof_config.rocm_path)
|
|
|
|
if cap.oshrun_version is not None:
|
|
oshrun_version_str = f"{cap.oshrun_version[0]}.{cap.oshrun_version[1]}"
|
|
else:
|
|
oshrun_version_str = "Not found"
|
|
|
|
oshrun_strips_str = (
|
|
"Yes (decoy '--' inserted)"
|
|
if cap.oshrun_strips_double_dash
|
|
else "No" if cap.oshrun_exec else "N/A"
|
|
)
|
|
|
|
if cap.amdsmi_version is not None:
|
|
amdsmi_version_str = f"{cap.amdsmi_version[0]}.{cap.amdsmi_version[1]}"
|
|
else:
|
|
amdsmi_version_str = "Not found"
|
|
|
|
if cap.amdgpu_version is not None:
|
|
amdgpu_version_str = (
|
|
f"{cap.amdgpu_version[0]}.{cap.amdgpu_version[1]}.{cap.amdgpu_version[2]}"
|
|
)
|
|
else:
|
|
amdgpu_version_str = "Not found"
|
|
|
|
# Rocprofiler SDK version
|
|
rocprofiler_sdk_version_str = (
|
|
f"{cap.rocprofiler_sdk_version[0]}.{cap.rocprofiler_sdk_version[1]}.{cap.rocprofiler_sdk_version[2]}"
|
|
if cap.rocprofiler_sdk_version
|
|
else "Not found"
|
|
)
|
|
|
|
W = 22 # label width for alignment
|
|
|
|
def _row(label: str, value) -> str:
|
|
return f" {label:<{W}}{value}"
|
|
|
|
def _subrow(label: str, value) -> str:
|
|
return f" {label:<{W}}{value}"
|
|
|
|
header = [
|
|
"",
|
|
"=" * 70,
|
|
"Test Configuration:",
|
|
"=" * 70,
|
|
_row("ROCm version:", rocm_version),
|
|
_row("ROCprof-SDK version:", rocprofiler_sdk_version_str),
|
|
_row("AMD-SMI version:", amdsmi_version_str),
|
|
_row("amdgpu version:", amdgpu_version_str),
|
|
_row("ROCm path:", rocprof_config.rocm_path),
|
|
_row("Is installed:", rocprof_config.is_installed),
|
|
_row("Output dir:", rocprof_config.test_output_dir),
|
|
_row("Validate ROCPD:", check_use_rocpd()),
|
|
_row("Validate Perfetto:", check_use_perfetto()),
|
|
"-" * 70,
|
|
"Core Executables:",
|
|
_row("Instrument:", rocprof_config.rocprofsys_instrument),
|
|
_row("Run:", rocprof_config.rocprofsys_run),
|
|
_row("Sample:", rocprof_config.rocprofsys_sample),
|
|
_row("Avail:", rocprof_config.rocprofsys_avail),
|
|
_row("Causal:", rocprof_config.rocprofsys_causal),
|
|
_row("Python:", rocprof_config.rocprofsys_python),
|
|
"-" * 70,
|
|
"Executables:",
|
|
_row("MPI:", cap.mpiexec_exec),
|
|
_subrow("Implementation:", cap.mpi_implementation),
|
|
_row("Julia:", cap.julia_exec),
|
|
_row("Oshrun:", cap.oshrun_exec),
|
|
_subrow("Version:", oshrun_version_str),
|
|
_subrow("Strips '--':", oshrun_strips_str),
|
|
_row("Offload tool:", offload_msg),
|
|
_row("Rocminfo:", rocminfo_path if rocminfo_path else rocminfo_err_msg),
|
|
"-" * 70,
|
|
"System Capabilities:",
|
|
_row("Detected num procs:", cap.num_procs),
|
|
_row("UCX available:", cap.ucx_availability),
|
|
_row("Perf event paranoid:", cap.perf_event_paranoid),
|
|
_row("CAP_SYS_ADMIN:", cap.cap_sys_admin),
|
|
_row("CAP_PERFMON:", cap.cap_perfmon),
|
|
_row("Perf events usable:", cap.perf_events_usable),
|
|
_row("Ptrace scope:", cap.ptrace_scope),
|
|
_row("Is inside docker:", rocprof_config.capabilities.is_inside_docker),
|
|
_row("PAPI available:", cap.papi_availability),
|
|
_row("AI NIC devices:", cap.ai_nic_devices),
|
|
_row("Default NIC:", cap.default_nic),
|
|
*(
|
|
lambda evts: (
|
|
[_row("PAPI NIC events:", evts[0])] + [_row("", e) for e in evts[1:]]
|
|
if evts
|
|
else [_row("PAPI NIC events:", "None")]
|
|
)
|
|
)(cap.papi_nic_events.split() if cap.papi_nic_events else []),
|
|
"-" * 70,
|
|
"GPU Information:",
|
|
_row("Available:", gpu_info.available),
|
|
_row("Architectures:", gpu_info.architectures or "None"),
|
|
_row("Device count:", gpu_info.device_count),
|
|
_row("Categories:", gpu_info.categories or "None"),
|
|
_row("XNACK support:", xnack_support),
|
|
"-" * 70,
|
|
"Directories:",
|
|
_row("Build dir:", rocprof_config.rocprofsys_build_dir),
|
|
_row("Lib dir:", rocprof_config.rocprofsys_lib_dir),
|
|
_row("Bin dir:", rocprof_config.rocprofsys_bin_dir),
|
|
_row("Tests dir:", rocprof_config.rocprofsys_tests_dir),
|
|
_row("Examples dir:", rocprof_config.rocprofsys_examples_dir),
|
|
_row("Validation dir:", rocprof_config.rocpd_validation_rules),
|
|
"-" * 70,
|
|
"Python:",
|
|
_row("Site packages:", rocprof_config.rocprofsys_site_packages),
|
|
]
|
|
if cap.supported_python_versions and cap.supported_python_executables:
|
|
for version, exe in zip(
|
|
cap.supported_python_versions,
|
|
cap.supported_python_executables,
|
|
):
|
|
header.append(_row(version, exe))
|
|
else:
|
|
header.append(
|
|
_row(
|
|
"Executables:",
|
|
"(no supported Python versions found — each version needs a "
|
|
"libpyrocprofsys.<IMPL>-<VERSION>-<ARCH>-<OS>-<ABI>.so in site-packages/rocprofsys)",
|
|
)
|
|
)
|
|
# Use fundamental system env to avoid verbose output
|
|
header.extend(["-" * 70, "System Environment:"])
|
|
for key, value in environment.fundamental_system_environment().items():
|
|
header.append(_row(f"{key}:", value))
|
|
header.extend(["=" * 70, ""])
|
|
return header
|
|
|
|
|
|
def add_marker_if(
|
|
item,
|
|
marker_to_add: str,
|
|
cond: Callable[[], bool] = lambda: True,
|
|
req_mark: Optional[str] = None,
|
|
skip_reason: Optional[str] = None,
|
|
unavailable_reason: Optional[Callable[[], Optional[str]]] = None,
|
|
) -> None:
|
|
"""Add a marker to a test item if:
|
|
- target_marker is present (or not specified)
|
|
- AND condition evaluates to True (lambda)
|
|
|
|
If ``unavailable_reason`` is set, it is called: ``None`` means add the marker;
|
|
a non-empty string means skip the test with that reason (preferred over cond/skip_reason).
|
|
|
|
If condition is False and skip_reason is provided, add a skip marker instead.
|
|
"""
|
|
if req_mark and not item.get_closest_marker(req_mark):
|
|
return
|
|
|
|
if unavailable_reason is not None:
|
|
msg = unavailable_reason()
|
|
if msg is None:
|
|
item.add_marker(getattr(pytest.mark, marker_to_add))
|
|
else:
|
|
item.add_marker(pytest.mark.skip(reason=msg))
|
|
return
|
|
|
|
if cond():
|
|
item.add_marker(getattr(pytest.mark, marker_to_add))
|
|
elif skip_reason:
|
|
item.add_marker(pytest.mark.skip(reason=skip_reason))
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def check_use_rocpd() -> bool:
|
|
"""Whether ROCpd is available for tests.
|
|
|
|
ROCpd requires:
|
|
- ROCPROFSYS_USE_ROCPD not set to OFF (default: ON)
|
|
- A valid GPU
|
|
- ROCm >= 7.0
|
|
"""
|
|
if os.environ.get("ROCPROFSYS_USE_ROCPD", "ON").upper() != "ON":
|
|
return False
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
except Exception as e:
|
|
pytest.exit(f"{e}")
|
|
gpu_info = get_gpu_info()
|
|
if not gpu_info.available:
|
|
return False
|
|
rocm_version = rocprof_config.rocm_version
|
|
return rocm_version is not None and rocm_version >= (7, 0, 0)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def check_use_perfetto() -> bool:
|
|
"""Whether Perfetto is available for tests.
|
|
|
|
Perfetto requires:
|
|
- ROCPROFSYS_VALIDATE_PERFETTO not set to OFF (default: ON)
|
|
- Perfetto Python module installed
|
|
"""
|
|
if os.environ.get("ROCPROFSYS_VALIDATE_PERFETTO", "ON").upper() != "ON":
|
|
return False
|
|
try:
|
|
import perfetto # noqa
|
|
|
|
return True
|
|
except ImportError:
|
|
return False
|
|
|
|
|
|
# The first call to this function MUST be performed in pytest_sessionstart
|
|
# as we need the --python-versions and --python-root-dirs options to be set
|
|
@lru_cache(maxsize=1)
|
|
def get_rocprof_config() -> RocprofsysConfig:
|
|
"""Return the rocprofiler-systems configuration."""
|
|
try:
|
|
pytest_config = getattr(pytest, "_config_ref", None)
|
|
python_versions = None
|
|
python_root_dirs = None
|
|
rocm_optional = False
|
|
if pytest_config:
|
|
ver_str = pytest_config.getoption("--python-versions", default=None)
|
|
dir_str = pytest_config.getoption("--python-root-dirs", default=None)
|
|
# When generating the CTestTestfile.cmake in TheRock, ROCm is not present
|
|
rocm_optional = (
|
|
pytest_config.getoption("--ctest-mode", default="off") == "generate"
|
|
)
|
|
if ver_str:
|
|
python_versions = [v.strip() for v in ver_str.split(";") if v.strip()]
|
|
if dir_str:
|
|
python_root_dirs = [
|
|
Path(d.strip()) for d in dir_str.split(";") if d.strip()
|
|
]
|
|
|
|
return discover_build_config(
|
|
python_versions=python_versions,
|
|
python_root_dirs=python_root_dirs,
|
|
rocm_optional=rocm_optional,
|
|
)
|
|
except Exception as e:
|
|
raise RuntimeError("Failed to get rocprofiler-systems configuration") from e
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_gpu_info() -> GPUInfo:
|
|
"""Return the GPU information."""
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
except Exception as e:
|
|
pytest.exit(f"{e}")
|
|
try:
|
|
return detect_gpu(rocprof_config.rocm_path)
|
|
except Exception as e:
|
|
pytest.exit(f"Failed to detect GPU: {e}")
|
|
|
|
|
|
def _run_cleanup() -> None:
|
|
"""Run cleanup of temp files and optionally the test output directory."""
|
|
import glob
|
|
import getpass
|
|
|
|
# Clean up temp files
|
|
for pattern in _cleanup_temp_patterns():
|
|
for filepath in glob.glob(pattern):
|
|
try:
|
|
p = Path(filepath)
|
|
if p.is_file() and p.owner() == getpass.getuser():
|
|
p.unlink()
|
|
print(f"Removed: {filepath}")
|
|
except (OSError, KeyError):
|
|
pass
|
|
|
|
# Clean up test output directory if ROCPROFSYS_KEEP_TEST_OUTPUT=0
|
|
if os.environ.get("ROCPROFSYS_KEEP_TEST_OUTPUT", "1") == "0":
|
|
try:
|
|
rocprof_config = get_rocprof_config()
|
|
output_dir = rocprof_config.test_output_dir
|
|
if output_dir.exists():
|
|
shutil.rmtree(output_dir)
|
|
print(f"Removed test output directory: {output_dir}")
|
|
except Exception as e:
|
|
print(f"Warning: Could not clean test output directory: {e}")
|
|
|
|
|
|
def _cleanup_temp_patterns() -> list[str]:
|
|
"""Return list of rocprofiler-systems temp file patterns to clean up."""
|
|
from rocprofsys.cache import resolve_username
|
|
|
|
tmpdir = os.environ.get("ROCPROFSYS_TMPDIR", os.environ.get("TMPDIR", "/tmp"))
|
|
dirs = ["/tmp"]
|
|
if tmpdir and not tmpdir.startswith("%") and tmpdir != "/tmp":
|
|
dirs.append(tmpdir)
|
|
|
|
user = resolve_username()
|
|
|
|
patterns = []
|
|
for d in dirs:
|
|
patterns.extend(
|
|
[
|
|
f"{d}/rocprof-sys-*.tmp",
|
|
f"{d}/rocprofsys-*.tmp",
|
|
f"{d}/buffered_storage*.bin",
|
|
f"{d}/metadata*.json",
|
|
f"{d}/perfetto-*.proto",
|
|
f"{d}/perfetto_trace*.proto",
|
|
f"{d}/hsa-*.tmp",
|
|
f"{d}/rocm-*.tmp",
|
|
f"{d}/hip-*.tmp",
|
|
f"{d}/*.inst",
|
|
f"{d}/causal-*.json",
|
|
f"{d}/experiments-*.coz",
|
|
f"{d}/core.*",
|
|
]
|
|
)
|
|
# The persistent capability cache lives in a per-user subdir
|
|
# (``<tmp>/$USER/rocprofsys-syscache-*.tmp``; see cache.py).
|
|
if user:
|
|
patterns.append(f"{d}/{user}/rocprofsys-*.tmp")
|
|
return patterns
|
|
|
|
|
|
# ============================================================================
|
|
#
|
|
# Fixtures
|
|
#
|
|
# ============================================================================
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Environment Fixtures
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def library_path(rocprof_config) -> str:
|
|
"""Computed LD_LIBRARY_PATH (rocprofsys libs + user override + ROCm LLVM libs)."""
|
|
return rocprof_config.get_library_path()
|
|
|
|
|
|
@pytest.fixture
|
|
def flat_env() -> dict[str, str]:
|
|
"""Environment variables for flat profile tests."""
|
|
return environment.flat_environment()
|
|
|
|
|
|
@pytest.fixture
|
|
def lock_env() -> dict[str, str]:
|
|
"""Environment variables for thread lock tracing tests."""
|
|
return environment.lock_environment()
|
|
|
|
|
|
@pytest.fixture
|
|
def perfetto_env() -> dict[str, str]:
|
|
"""Environment variables for perfetto-only tests."""
|
|
return environment.perfetto_environment()
|
|
|
|
|
|
@pytest.fixture
|
|
def timemory_env() -> dict[str, str]:
|
|
"""Environment variables for timemory-only tests."""
|
|
return environment.timemory_environment()
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Session-scoped Fixtures
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def get_test_num_threads(rocprof_config) -> int:
|
|
"""Get the number of threads for the test."""
|
|
num_threads = rocprof_config.capabilities.num_procs + (
|
|
rocprof_config.capabilities.num_procs // 2
|
|
)
|
|
if num_threads > 12:
|
|
return 12
|
|
return num_threads
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def rocprof_config() -> RocprofsysConfig:
|
|
"""Session-wide rocprofiler-systems configuration.
|
|
|
|
Discovers build directory and creates configuration object.
|
|
Can be overridden with ROCPROFSYS_BUILD_DIR environment variable.
|
|
"""
|
|
return get_rocprof_config()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def gpu_info() -> GPUInfo:
|
|
"""Session-wide GPU information.
|
|
|
|
Detects available GPUs and their capabilities.
|
|
"""
|
|
return get_gpu_info()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def tests_dir(rocprof_config) -> Path:
|
|
"""Path to tests directory."""
|
|
return rocprof_config.rocprofsys_tests_dir
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def validation_rules_dir(rocprof_config) -> Path:
|
|
"""Path to validation rules directory."""
|
|
return rocprof_config.rocpd_validation_rules
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Module-scoped Fixtures
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def test_output_base(rocprof_config) -> Path:
|
|
"""Base directory for test outputs"""
|
|
output_dir = rocprof_config.test_output_dir
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
return output_dir
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Function-scoped Fixtures
|
|
# ----------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def create_config_file(test_output_dir) -> Path:
|
|
"""Create a config file for a test based on env vars and return Path.
|
|
|
|
Filters out environment-only settings that should not be written to config files
|
|
"""
|
|
# Settings that should only be in environment, not config files
|
|
env_only_pattern = re.compile(
|
|
r"ROCPROFSYS_(CI|CI_TIMEOUT|MODE|USE_MPIP|DEBUG_[A-Z_]+|"
|
|
r"FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)$"
|
|
)
|
|
|
|
def _create_config_file(
|
|
env: dict[str, str],
|
|
name: Optional[str] = "config.cfg",
|
|
skip_filter: bool = False,
|
|
) -> Path:
|
|
config_file = test_output_dir / name
|
|
content = "# auto-generated by pytest\n\n"
|
|
|
|
if skip_filter:
|
|
config_vars = {k: v for k, v in env.items() if k != "ROCPROFSYS_CONFIG_FILE"}
|
|
else:
|
|
# Only write ROCPROFSYS_* settings to config file, excluding env-only settings
|
|
# Non-ROCPROFSYS vars (OMP_*, LD_LIBRARY_PATH, etc.) should stay as env vars only
|
|
# Also exclude ROCPROFSYS_CONFIG_FILE to avoid self-reference
|
|
config_vars = {
|
|
k: v
|
|
for k, v in env.items()
|
|
if k.startswith("ROCPROFSYS_")
|
|
and not env_only_pattern.match(k)
|
|
and k != "ROCPROFSYS_CONFIG_FILE"
|
|
}
|
|
|
|
content += "\n".join(f"{k}={v}" for k, v in config_vars.items())
|
|
config_file.write_text(content)
|
|
return config_file
|
|
|
|
return _create_config_file
|
|
|
|
|
|
@pytest.fixture
|
|
def collect_result(request) -> Callable:
|
|
"""Fixture to collect test results for display.
|
|
|
|
Handled by the `run_test` fixture
|
|
|
|
Manual usage in tests:
|
|
result = runner.run()
|
|
collect_result(result)
|
|
"""
|
|
|
|
def _collect(result):
|
|
request.node.stash[_result_key] = result
|
|
|
|
return _collect
|
|
|
|
|
|
@pytest.fixture
|
|
def test_output_dir(
|
|
test_output_base: Path,
|
|
request: pytest.FixtureRequest,
|
|
) -> Generator[Path, None, None]:
|
|
"""Unique output directory for each test.
|
|
|
|
Creates a directory named after the test and cleans up on success.
|
|
On failure, the directory is preserved for debugging.
|
|
Directory is removed if it is empty.
|
|
|
|
Cleanup Order:
|
|
1. Test setup: Directory is created
|
|
2. Test body: Runner executes, output files are written
|
|
3. Test body: Validation happens on output files
|
|
4. Test body: Assertions complete
|
|
5. Test teardown: This fixture cleans up the directory (AFTER yield)
|
|
|
|
This ensures validation always has access to output files.
|
|
"""
|
|
output_dir = test_output_base / request.node.name
|
|
|
|
if output_dir.exists():
|
|
shutil.rmtree(output_dir)
|
|
output_dir.mkdir(parents=True)
|
|
|
|
yield output_dir # Test body executes here (including validation)
|
|
|
|
# === CLEANUP PHASE (runs AFTER test body completes) ===
|
|
|
|
# If the output directory is empty, remove it
|
|
if output_dir.exists() and not any(output_dir.iterdir()):
|
|
shutil.rmtree(output_dir)
|
|
|
|
keep_output = os.environ.get("ROCPROFSYS_KEEP_TEST_OUTPUT", "1") == "1"
|
|
test_failed = hasattr(request.node, "rep_call") and request.node.rep_call.failed
|
|
|
|
if keep_output or test_failed or not output_dir.exists():
|
|
return
|
|
|
|
# Remove all files in the output directory, then the directory itself
|
|
# unless the preserve marker is present
|
|
|
|
to_preserve = []
|
|
preserve_marker = request.node.get_closest_marker("preserve")
|
|
if preserve_marker and preserve_marker.args:
|
|
for fname in preserve_marker.args:
|
|
preserved = output_dir / fname
|
|
to_preserve.append(preserved)
|
|
|
|
for entry in output_dir.iterdir():
|
|
if entry not in to_preserve:
|
|
safe_remove(entry)
|
|
|
|
# Remove the output directory if empty
|
|
if not any(output_dir.iterdir()):
|
|
shutil.rmtree(output_dir)
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def apply_rocpd_marker(request):
|
|
"""Automatically add ROCpd env vars based on marker.
|
|
|
|
Usage:
|
|
@pytest.mark.rocpd("<env name>")
|
|
"""
|
|
if not check_use_rocpd():
|
|
return
|
|
|
|
marker = request.node.get_closest_marker("rocpd")
|
|
if not marker or not marker.args:
|
|
return
|
|
|
|
# First arg is fixture name
|
|
env_fixture_name = marker.args[0]
|
|
|
|
try:
|
|
env = request.getfixturevalue(env_fixture_name)
|
|
except pytest.FixtureLookupError:
|
|
return
|
|
|
|
# Add ROCpd base env
|
|
env["ROCPROFSYS_USE_ROCPD"] = "ON"
|
|
|
|
|
|
def _print_subtest_output(request, subtest_name: str, output: str) -> None:
|
|
"""Print subtest validation output for important subtests when in CTest run mode."""
|
|
if request.config.getoption("--ctest-mode", default="off") == "run" and output:
|
|
print(f"\n--- {subtest_name} ---\n{output}\n", flush=True)
|
|
|
|
|
|
# Contains a set of kwargs accepted for a given (function, mode) pair.
|
|
_FUNCTION_ALLOWED_KWARGS: dict[str, dict[str, set[str]]] = {
|
|
"run_test": {
|
|
"baseline": {"command"},
|
|
"sampling": {"sampling_args"},
|
|
"binary_rewrite": {"binary_rewrite_args", "cleanup_on_success"},
|
|
"runtime_instrument": {"runtime_instrument_args"},
|
|
"sys_run": {"sys_run_args"},
|
|
"causal": {"causal_args", "causal_mode"},
|
|
"python": {"python_version", "profile_args", "annotated", "standalone"},
|
|
},
|
|
"assert_regex": {
|
|
"baseline": {"baseline_pass_regex", "baseline_fail_regex"},
|
|
"sampling": {"sampling_pass_regex", "sampling_fail_regex"},
|
|
"binary_rewrite": {"binary_rewrite_pass_regex", "binary_rewrite_fail_regex"},
|
|
"runtime_instrument": {
|
|
"runtime_instrument_pass_regex",
|
|
"runtime_instrument_fail_regex",
|
|
},
|
|
"sys_run": {"sys_run_pass_regex", "sys_run_fail_regex"},
|
|
"causal": {"causal_pass_regex", "causal_fail_regex"},
|
|
"python": {"python_pass_regex", "python_fail_regex"},
|
|
},
|
|
}
|
|
|
|
|
|
def _filter_kwargs(function: str, mode: str, **kwargs: Any) -> dict[str, Any]:
|
|
"""Filter ``kwargs`` to those accepted by ``function`` for ``mode``.
|
|
|
|
This also verifies that the kwargs passed are valid for the given function.
|
|
If a kwarg is not valid, pytest.fail is called.
|
|
|
|
Returns:
|
|
A new dict containing only kwargs valid for ``(function, mode)``.
|
|
"""
|
|
allowed_per_mode = _FUNCTION_ALLOWED_KWARGS.get(function)
|
|
if allowed_per_mode is None:
|
|
pytest.fail(
|
|
f"_filter_kwargs called with unknown function '{function}'. "
|
|
f"Expected one of: {sorted(_FUNCTION_ALLOWED_KWARGS.keys())}."
|
|
)
|
|
|
|
mode_key = mode.replace("-", "_")
|
|
allowed_for_mode = allowed_per_mode.get(mode_key)
|
|
if allowed_for_mode is None:
|
|
pytest.fail(
|
|
f"Unknown mode '{mode}' for '{function}'. "
|
|
f"Expected one of: {sorted(allowed_per_mode.keys())}."
|
|
)
|
|
|
|
# Union of every kwarg accepted by any mode of this function. Anything
|
|
# outside this set is considered a typo and an error is raised.
|
|
all_known_for_function: set[str] = set().union(*allowed_per_mode.values())
|
|
unknown = set(kwargs) - all_known_for_function
|
|
if unknown:
|
|
pytest.fail(
|
|
f"{function}: unknown kwargs {sorted(unknown)}. "
|
|
f"Valid kwargs across all modes: {sorted(all_known_for_function)}."
|
|
)
|
|
|
|
return {k: v for k, v in kwargs.items() if k in allowed_for_mode}
|
|
|
|
|
|
# ============================================================================
|
|
# Base Test Class
|
|
# ============================================================================
|
|
|
|
|
|
class RocprofsysTest:
|
|
"""Base class that auto-captures parametrized values and common fixtures onto self."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _setup(
|
|
self,
|
|
run_test,
|
|
assert_regex,
|
|
assert_perfetto,
|
|
assert_rocpd,
|
|
assert_unified_memory_output,
|
|
assert_causal_json,
|
|
assert_file_exists,
|
|
assert_timemory,
|
|
assert_file_regex,
|
|
get_test_num_threads,
|
|
test_output_dir,
|
|
library_path,
|
|
):
|
|
|
|
self.run_test = run_test
|
|
self.assert_regex = assert_regex
|
|
self.assert_perfetto = assert_perfetto
|
|
self.assert_rocpd = assert_rocpd
|
|
self.assert_unified_memory_output = assert_unified_memory_output
|
|
self.assert_causal_json = assert_causal_json
|
|
self.assert_file_exists = assert_file_exists
|
|
self.assert_timemory = assert_timemory
|
|
self.assert_file_regex = assert_file_regex
|
|
self.num_threads = get_test_num_threads
|
|
self.test_output_dir = test_output_dir
|
|
self.library_path = library_path
|
|
|
|
|
|
# ============================================================================
|
|
# Test run and assertion fixtures
|
|
# ============================================================================
|
|
|
|
|
|
@pytest.fixture
|
|
def run_test(
|
|
request,
|
|
collect_result,
|
|
rocprof_config,
|
|
gpu_info,
|
|
test_output_dir,
|
|
):
|
|
"""Unified fixture to run any test runner type and handle pytest logic.
|
|
If a rocprof-sys binary is provided, uses "base_binary_environment" instead of "base_environment".
|
|
|
|
Args:
|
|
runner_type: One of "baseline", "sampling", "binary_rewrite",
|
|
"runtime_instrument", "sys_run"
|
|
target: Target executable name
|
|
run_args: Arguments passed to the target executable
|
|
env: Environment variables dict
|
|
launcher: Launcher to use (mpi or shmem)
|
|
|
|
num_procs: Number of processes (0 = disabled)
|
|
working_directory: Custom working directory
|
|
check_target_arch: If True, checks if the target supports the current system architectures (default: False)
|
|
Note: This requires @pytest.mark.gpu to be present
|
|
skip_on_error: If True, pytest.skip on non-zero return code (default: False = fail)
|
|
fail_on_pass: If True, pytest.fail on success and pytest.pass on failure (default: False)
|
|
fail_on_not_found: If True, pytest.fail when binary not found (default: False = skip)
|
|
fail_message: Custom failure message (default: "{runner_type} test failed: {output}")
|
|
no_base_env: If true, don't use the base environment (default: False)
|
|
**kwargs: Additional runner-specific arguments (see _FUNCTION_ALLOWED_KWARGS for valid kwargs)
|
|
|
|
Returns:
|
|
TestResult for further assertions
|
|
"""
|
|
|
|
def _run_test(
|
|
runner_type: str,
|
|
target: str,
|
|
env: Optional[dict[str, str]] = None,
|
|
run_args: Optional[list[str]] = None,
|
|
pre_run_args: Optional[list[str]] = None,
|
|
launcher: Optional[BaselineRunner.Launcher | str] = None,
|
|
num_procs: int = 0,
|
|
working_directory: Optional[Path] = None,
|
|
check_target_arch: bool = False,
|
|
skip_on_error: bool = False,
|
|
fail_on_pass: bool = False,
|
|
fail_on_not_found: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
no_base_env: bool = False,
|
|
**kwargs,
|
|
) -> TestResult:
|
|
filtered_kwargs = _filter_kwargs("run_test", runner_type, **kwargs)
|
|
|
|
if num_procs > 0 and launcher is None:
|
|
pytest.fail(
|
|
f"num_procs={num_procs} was provided but no launcher was set. "
|
|
f"Pass launcher='<launcher_name>' alongside num_procs."
|
|
)
|
|
|
|
if runner_type == "causal" and "causal_mode" not in filtered_kwargs:
|
|
pytest.exit("causal_mode is required for causal tests", returncode=1)
|
|
|
|
runner_class = ROCPROFSYS_RUNNER_CLASSES.get(runner_type)
|
|
if not runner_class:
|
|
pytest.fail(
|
|
f"Invalid runner type: {runner_type}. Use: {list(ROCPROFSYS_RUNNER_CLASSES.keys())}"
|
|
)
|
|
|
|
# For GPU tests, ensure that the target supports at least one of the current system architectures
|
|
if request.node.get_closest_marker("gpu") and check_target_arch:
|
|
try:
|
|
target_path = rocprof_config.get_target_executable(target)
|
|
target_archs = get_target_gpu_arch(rocprof_config.rocm_path, target_path)
|
|
system_archs = gpu_info.architectures
|
|
if not any(arch in target_archs for arch in system_archs):
|
|
pytest.skip(
|
|
f"{target} does not support any of the current system architectures. "
|
|
f"{target} architectures: {target_archs}, system architectures: {system_archs}"
|
|
)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
env = env.copy() if env else {}
|
|
|
|
# Timeout: ROCPROFSYS_CI_TIMEOUT env, else @pytest.mark.timeout, else default
|
|
ci_timeout_env = os.environ.get("ROCPROFSYS_CI_TIMEOUT")
|
|
if ci_timeout_env is not None:
|
|
# Shell-exported value: drives the subprocess timeout below and is
|
|
# already carried by the user env layer. Do NOT echo it into the
|
|
# test layer, or it would mask the real base default in dumps.
|
|
timeout = int(ci_timeout_env)
|
|
elif request.node.get_closest_marker("timeout"):
|
|
timeout = request.node.get_closest_marker("timeout").args[0]
|
|
env["ROCPROFSYS_CI_TIMEOUT"] = str(timeout)
|
|
else:
|
|
timeout = 300
|
|
env["ROCPROFSYS_CI_TIMEOUT"] = str(timeout)
|
|
|
|
# Verify that MPI is available for "mpi_optional" tests
|
|
if request.node.get_closest_marker("mpi_optional") and num_procs > 0:
|
|
if not request.node.get_closest_marker("mpi"):
|
|
num_procs = 0
|
|
|
|
try:
|
|
runner = runner_class(
|
|
config=rocprof_config,
|
|
target=target,
|
|
output_dir=test_output_dir,
|
|
run_args=run_args,
|
|
pre_run_args=pre_run_args,
|
|
env=env,
|
|
timeout=timeout,
|
|
launcher=launcher,
|
|
num_procs=num_procs,
|
|
working_directory=working_directory,
|
|
no_base_env=no_base_env,
|
|
**filtered_kwargs,
|
|
)
|
|
except FileNotFoundError:
|
|
if fail_on_not_found:
|
|
pytest.fail(f"{target} binary not found")
|
|
else:
|
|
pytest.skip(f"{target} binary not found")
|
|
|
|
result = runner.run()
|
|
collect_result(result)
|
|
|
|
if not result.success and not fail_on_pass:
|
|
short_msg = fail_message or f"{runner_type} test failed"
|
|
if skip_on_error:
|
|
pytest.skip(short_msg)
|
|
else:
|
|
pytest.fail(short_msg)
|
|
|
|
if fail_on_pass and result.success:
|
|
pytest.fail(f"{runner_type} test passed unexpectedly")
|
|
|
|
return result
|
|
|
|
return _run_test
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_regex(subtests):
|
|
"""Fixture that returns an assert_regex function.
|
|
|
|
Args:
|
|
result: TestResult from run_test
|
|
mode: Optional runner type (e.g., "binary_rewrite", "sys_run"). If provided, looks up
|
|
mode-specific regexes from kwargs (see _FUNCTION_ALLOWED_KWARGS for valid kwargs)
|
|
subtest_name: Name shown in subtest output (defaults to "Regex validation")
|
|
pass_regex: Explicit list of pass regex patterns (used if mode is None or no mode-specific found)
|
|
fail_regex: Explicit list of fail regex patterns (used if mode is None or no mode-specific found)
|
|
use_abort_fail_regex: Whether to validate against ROCPROFSYS_ABORT_FAIL_REGEX (default: True)
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
**kwargs: Mode-specific regexes (see _FUNCTION_ALLOWED_KWARGS for valid kwargs)
|
|
"""
|
|
|
|
def _assert_regex(
|
|
result: TestResult,
|
|
mode: Optional[str] = None,
|
|
subtest_name: str = "Regex validation",
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
use_abort_fail_regex: bool = True,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
**kwargs,
|
|
) -> None:
|
|
|
|
if mode is None and kwargs:
|
|
pytest.fail(
|
|
f"assert_regex received mode-specific kwargs {sorted(kwargs)} but no "
|
|
f"'mode' was provided. Pass mode=... so they can be resolved, or use "
|
|
f"pass_regex/fail_regex directly."
|
|
)
|
|
|
|
if mode is not None:
|
|
filtered = _filter_kwargs("assert_regex", mode, **kwargs)
|
|
mode_key = mode.replace("-", "_")
|
|
mode_pass_regex = filtered.get(f"{mode_key}_pass_regex")
|
|
if mode_pass_regex is not None:
|
|
pass_regex = mode_pass_regex
|
|
mode_fail_regex = filtered.get(f"{mode_key}_fail_regex")
|
|
if mode_fail_regex is not None:
|
|
fail_regex = mode_fail_regex
|
|
|
|
with subtests.test(subtest_name):
|
|
validation = validate_regex(
|
|
result, pass_regex, fail_regex, use_abort_fail_regex
|
|
)
|
|
if not validation.is_valid:
|
|
msg = fail_message or f"Regex validation failed: {validation.message}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
|
|
return _assert_regex
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_file_regex(subtests):
|
|
"""Variant of assert_regex that validates against a file."""
|
|
|
|
def _assert_file_regex(
|
|
file_path: Path,
|
|
subtest_name: str = "File regex validation",
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
use_abort_fail_regex: bool = True,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
validation = validate_file_regex(
|
|
file_path,
|
|
pass_regex,
|
|
fail_regex,
|
|
use_abort_fail_regex,
|
|
)
|
|
|
|
if not validation.is_valid:
|
|
msg = (
|
|
fail_message or f"File regex validation failed: {validation.message}"
|
|
)
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
|
|
return _assert_file_regex
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_perfetto(subtests, tests_dir, request, test_output_dir):
|
|
"""Fixture that returns an assert_perfetto function.
|
|
|
|
Trace validation kwargs (``categories``, ``labels``, ``counts``, ``depths``,
|
|
``label_substrings``, etc.) are forwarded to
|
|
``validate_perfetto_trace``; see that function's docstring.
|
|
|
|
Args not from validate_perfetto_trace:
|
|
subtest_name: Name shown in subtest output (defaults to "Perfetto validation")
|
|
perfetto_file: (Optional) Name of the perfetto file in the test output directory (e.g., for merged.proto)
|
|
pass_regex: (Optional) Regex patterns that must be found in validation.stdout
|
|
fail_regex: (Optional) Regex patterns that must NOT be found in validation.stdout
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
"""
|
|
|
|
def _assert_perfetto(
|
|
result: TestResult,
|
|
subtest_name: str = "Perfetto validation",
|
|
perfetto_file: Optional[Path] = None,
|
|
categories: Optional[list[str]] = None,
|
|
labels: Optional[list[str]] = None,
|
|
counts: Optional[list[int]] = None,
|
|
depths: Optional[list[int]] = None,
|
|
label_substrings: Optional[list[str]] = None,
|
|
counter_names: Optional[list[str]] = None,
|
|
check_counter_pairing: bool = False,
|
|
key_names: Optional[list[str]] = None,
|
|
key_counts: Optional[list[int]] = None,
|
|
trace_processor_path: Optional[Path] = None,
|
|
print_output: bool = True,
|
|
timeout: int = 120,
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
if not check_use_perfetto():
|
|
pytest.skip("Perfetto is disabled")
|
|
|
|
# Perfetto file check
|
|
if perfetto_file is not None:
|
|
perfetto = Path(test_output_dir) / perfetto_file
|
|
else:
|
|
perfetto = result.perfetto_file
|
|
if perfetto is None:
|
|
pytest.fail("No Perfetto trace file was produced by the run")
|
|
if not perfetto.exists():
|
|
pytest.fail(f"Perfetto trace file {perfetto} not found")
|
|
|
|
validation = validate_perfetto_trace(
|
|
perfetto,
|
|
tests_dir=tests_dir,
|
|
categories=categories,
|
|
labels=labels,
|
|
counts=counts,
|
|
depths=depths,
|
|
label_substrings=label_substrings,
|
|
counter_names=counter_names,
|
|
check_counter_pairing=check_counter_pairing,
|
|
key_names=key_names,
|
|
key_counts=key_counts,
|
|
trace_processor_path=trace_processor_path,
|
|
print_output=print_output,
|
|
timeout=timeout,
|
|
)
|
|
output = f"Command: {validation.command}\n\n{validation.message}"
|
|
if not validation.is_valid:
|
|
msg = fail_message or f"Perfetto validation failed:\n{output}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
if pass_regex:
|
|
for pattern in pass_regex:
|
|
if not re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Pass regex not found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
if fail_regex:
|
|
for pattern in fail_regex:
|
|
if re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Fail regex found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
# On success echo only the command: validation.message may contain the
|
|
# full "-p"/--print perfetto dump, which floods ctest/CI logs. Failures
|
|
# above still surface the complete output.
|
|
_print_subtest_output(request, subtest_name, f"Command: {validation.command}")
|
|
|
|
return _assert_perfetto
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_rocpd(subtests, tests_dir, request):
|
|
"""Fixture that returns an assert_rocpd function.
|
|
|
|
Must be used with @pytest.mark.rocpd("<env fixture name>")
|
|
|
|
Args not from validate_rocpd_database:
|
|
subtest_name: Name shown in subtest output (defaults to "ROCpd validation")
|
|
pass_regex: (Optional) Regex patterns that must be found in validation.stdout
|
|
fail_regex: (Optional) Regex patterns that must NOT be found in validation.stdout
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
gpu_category_to_skip: GPU categories to skip tagged validation queries for
|
|
(instinct, radeon, apu). Omit or pass empty to run all queries
|
|
"""
|
|
|
|
def _assert_rocpd(
|
|
result: TestResult,
|
|
subtest_name: str = "ROCpd validation",
|
|
rules_files: Optional[list[Path]] = None,
|
|
timeout: int = 60,
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
gpu_category_to_skip: Optional[list[str]] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
if not check_use_rocpd():
|
|
pytest.skip("ROCpd is disabled")
|
|
rocpd_files = result.rocpd_files
|
|
if not rocpd_files:
|
|
pytest.fail("ROCpd database not created")
|
|
|
|
existing_rules = None
|
|
if rules_files is not None:
|
|
existing_rules = [r for r in rules_files if r.exists()]
|
|
if not existing_rules:
|
|
pytest.fail("No validation rules found")
|
|
|
|
def validate_candidate(rocpd_file: Path) -> ValidationResult:
|
|
return validate_rocpd_database(
|
|
rocpd_file,
|
|
tests_dir=tests_dir,
|
|
rules_files=existing_rules,
|
|
timeout=timeout,
|
|
gpu_category_to_skip=gpu_category_to_skip,
|
|
)
|
|
|
|
passing_output, failures, global_failure = _validate_rocpd_candidates(
|
|
rocpd_files,
|
|
validate_candidate,
|
|
pass_regex=pass_regex,
|
|
fail_regex=fail_regex,
|
|
)
|
|
|
|
if global_failure is not None:
|
|
msg = fail_message or f"ROCpd validation failed:\n{global_failure}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg, pytrace=False)
|
|
elif passing_output is None:
|
|
output = "\n\n--- Next ROCpd candidate ---\n\n".join(failures)
|
|
msg = fail_message or f"ROCpd validation failed:\n{output}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg, pytrace=False)
|
|
_print_subtest_output(request, subtest_name, passing_output)
|
|
|
|
return _assert_rocpd
|
|
|
|
|
|
def _validate_rocpd_candidates(
|
|
rocpd_files: list[Path],
|
|
validate_candidate: Callable[[Path], ValidationResult],
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
) -> tuple[Optional[str], list[str], Optional[str]]:
|
|
"""Validate ROCpd candidates and return the first passing output.
|
|
|
|
Multi-process runs can emit multiple ROCpd databases. Some rank-local
|
|
databases may not contain the GPU rows required by a rule set, so the
|
|
validation succeeds if any emitted candidate fully validates. A fail regex
|
|
match is a global failure and stops validation immediately.
|
|
"""
|
|
failures: list[str] = []
|
|
|
|
for rocpd_file in rocpd_files:
|
|
validation = validate_candidate(rocpd_file)
|
|
output = f"Command: {validation.command}\n\n{validation.message}"
|
|
if fail_regex:
|
|
for pattern in fail_regex:
|
|
if re.search(pattern, validation.stdout):
|
|
return None, failures, f"Fail regex found: {pattern}\n{output}"
|
|
if not validation.is_valid:
|
|
failures.append(output)
|
|
continue
|
|
|
|
regex_failure = None
|
|
if pass_regex:
|
|
for pattern in pass_regex:
|
|
if not re.search(pattern, validation.stdout):
|
|
regex_failure = f"Pass regex not found: {pattern}"
|
|
break
|
|
if regex_failure is not None:
|
|
failures.append(f"{regex_failure}\n{output}")
|
|
continue
|
|
|
|
return output, failures, None
|
|
|
|
return None, failures, None
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_timemory(subtests, tests_dir, request):
|
|
"""Fixture that returns an assert_timemory function.
|
|
|
|
Args not from validate_timemory_json:
|
|
subtest_name: Name shown in subtest output (defaults to "Timemory validation")
|
|
pass_regex: (Optional) Regex patterns that must be found in validation.stdout
|
|
fail_regex: (Optional) Regex patterns that must NOT be found in validation.stdout
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
"""
|
|
|
|
def _assert_timemory(
|
|
result: TestResult,
|
|
file_name: str,
|
|
metric: str,
|
|
subtest_name: str = "Timemory validation",
|
|
labels: Optional[list[str]] = None,
|
|
counts: Optional[list[int]] = None,
|
|
depths: Optional[list[int]] = None,
|
|
print_output: bool = True,
|
|
timeout: int = 60,
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
timemory_file = result.output_dir / file_name
|
|
if not timemory_file.exists():
|
|
pytest.fail(f"Timemory file not found: {timemory_file}")
|
|
validation = validate_timemory_json(
|
|
json_path=timemory_file,
|
|
tests_dir=tests_dir,
|
|
metric=metric,
|
|
labels=labels,
|
|
counts=counts,
|
|
depths=depths,
|
|
print_output=print_output,
|
|
timeout=timeout,
|
|
)
|
|
output = f"Command: {validation.command}\n\n{validation.message}"
|
|
if not validation.is_valid:
|
|
msg = fail_message or f"Timemory validation failed:\n{output}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
if pass_regex:
|
|
for pattern in pass_regex:
|
|
if not re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Pass regex not found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
if fail_regex:
|
|
for pattern in fail_regex:
|
|
if re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Fail regex found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
_print_subtest_output(request, subtest_name, output)
|
|
|
|
return _assert_timemory
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_file_exists(subtests):
|
|
"""Fixture that returns an assert_file_exists function.
|
|
|
|
Args not from validate_file_exists:
|
|
subtest_name: Name shown in subtest output (defaults to "File existence validation")
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
"""
|
|
|
|
def _assert_file_exists(
|
|
path: Path | list[Path],
|
|
description: str = "File",
|
|
subtest_name: str = "File existence validation",
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
paths = [path] if isinstance(path, Path) else path
|
|
with subtests.test(subtest_name):
|
|
for p in paths:
|
|
validation = validate_file_exists(p, description)
|
|
if not validation.is_valid:
|
|
msg = (
|
|
fail_message
|
|
or f"File existence validation failed: {validation.message}"
|
|
)
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
|
|
return _assert_file_exists
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_unified_memory_output(subtests, tests_dir, request):
|
|
"""Fixture that returns an assert_unified_memory_output function."""
|
|
|
|
def _assert_unified_memory_output(
|
|
result: TestResult,
|
|
subtest_name: str = "Unified-memory output validation",
|
|
timeout: int = 60,
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
validation = validate_unified_memory_outputs(
|
|
result.output_dir,
|
|
tests_dir=tests_dir,
|
|
timeout=timeout,
|
|
)
|
|
output = f"Command: {validation.command}\n\n{validation.message}"
|
|
if not validation.is_valid:
|
|
msg = fail_message or f"Unified-memory validation failed:\n{output}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
if pass_regex:
|
|
for pattern in pass_regex:
|
|
if not re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Pass regex not found: {pattern}\n{output}",
|
|
pytrace=False,
|
|
)
|
|
if fail_regex:
|
|
for pattern in fail_regex:
|
|
if re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Fail regex found: {pattern}\n{output}",
|
|
pytrace=False,
|
|
)
|
|
_print_subtest_output(request, subtest_name, output)
|
|
|
|
return _assert_unified_memory_output
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_causal_json(subtests, tests_dir, request):
|
|
"""Fixture that returns an assert_causal_json function.
|
|
|
|
Args not from validate_causal_json:
|
|
pass_regex: (Optional) Regex patterns that must be found in validation.stdout
|
|
fail_regex: (Optional) Regex patterns that must NOT be found in validation.stdout
|
|
skip_on_fail: If True, skip instead of fail when validation fails
|
|
fail_message: Custom message for failure (defaults to validation message)
|
|
"""
|
|
|
|
def _assert_causal_json(
|
|
result: TestResult,
|
|
file_name: str,
|
|
subtest_name: str = "Causal JSON validation",
|
|
ci_mode: bool = False,
|
|
additional_args: Optional[list[str]] = None,
|
|
timeout: int = 60,
|
|
pass_regex: Optional[list[str]] = None,
|
|
fail_regex: Optional[list[str]] = None,
|
|
skip_on_fail: bool = False,
|
|
fail_message: Optional[str] = None,
|
|
) -> None:
|
|
with subtests.test(subtest_name):
|
|
causal_file = result.output_dir / file_name
|
|
if not causal_file.exists():
|
|
pytest.fail(f"Causal JSON file not found: {causal_file}")
|
|
|
|
validation = validate_causal_json(
|
|
json_path=causal_file,
|
|
tests_dir=tests_dir,
|
|
ci_mode=ci_mode,
|
|
additional_args=additional_args,
|
|
timeout=timeout,
|
|
)
|
|
output = f"Command: {validation.command}\n\n{validation.message}"
|
|
if not validation.is_valid:
|
|
if fail_message:
|
|
msg = f"{fail_message}:\n{output}"
|
|
else:
|
|
msg = f"Causal JSON validation failed:\n{output}"
|
|
if skip_on_fail:
|
|
pytest.skip(msg)
|
|
else:
|
|
pytest.fail(msg)
|
|
|
|
if pass_regex:
|
|
for pattern in pass_regex:
|
|
if not re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Pass regex not found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
|
|
if fail_regex:
|
|
for pattern in fail_regex:
|
|
if re.search(pattern, validation.stdout):
|
|
pytest.fail(
|
|
f"Fail regex found: {pattern}\n{output}", pytrace=False
|
|
)
|
|
_print_subtest_output(request, subtest_name, output)
|
|
|
|
return _assert_causal_json
|