TheRock/external-builds/pytorch/run_pytorch_smoke_tests.py

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

"""PyTorch ROCm Smoke Tests Runner.

This script runs PyTorch smoke tests using pytest.

Usage Examples
--------------
Basic usage (auto-detect GPU):
    $ python run_pytorch_smoke_tests.py

Specify GPU family:
    $ python run_pytorch_smoke_tests.py --amdgpu-family gfx942

Pass additional pytest arguments after "--":
    $ python run_pytorch_smoke_tests.py -- -m "slow"
    $ python run_pytorch_tests.py -- --tb=short -x
"""

import argparse
import os
import subprocess
import sys

from pathlib import Path

from pytorch_utils import (
    get_unique_supported_devices,
    reconcile_agent_visibility_env,
    set_gpu_execution_policy,
)

THIS_SCRIPT_DIR = Path(__file__).resolve().parent


def setup_env() -> None:
    """Set up environment variables required for the smoke tests."""
    reconcile_agent_visibility_env()


def cmd_arguments(argv: list[str]) -> tuple[argparse.Namespace, list[str]]:
    """Parse command line arguments.

    Args:
        argv: Command line arguments (without program name).

    Returns:
        Tuple of (parsed args, passthrough pytest args passed after "--").
    """
    # Extract passthrough pytest args after "--"
    try:
        rest_pos = argv.index("--")
    except ValueError:
        passthrough_pytest_args = []
    else:
        passthrough_pytest_args = argv[rest_pos + 1 :]
        argv = argv[:rest_pos]

    parser = argparse.ArgumentParser(
        description="""
Runs PyTorch smoke-tests for AMD GPUs.
All arguments after "--" are passed directly to pytest (e.g. -p no:cacheprovider).
"""
    )

    parser.add_argument(
        "--amdgpu-family",
        type=str,
        default=os.getenv("AMDGPU_FAMILY", ""),
        required=False,
        help="""Amdgpu family (e.g. "gfx942").
Select (potentially) additional tests to be skipped based on the amdgpu family""",
    )

    args = parser.parse_args(argv)

    return args, passthrough_pytest_args


def main() -> int:
    """Main entry point for the PyTorch smoke-tests runner.

    Returns:
        Exit code from pytest (0 for success, non-zero for failures).
        Returns non-zero if any device's tests fail.
    """
    try:
        args, passthrough_pytest_args = cmd_arguments(sys.argv[1:])

        # Assumes that the smoke-tests are located in the same directory as this script
        smoke_tests_dir = THIS_SCRIPT_DIR / "smoke-tests"

        if not smoke_tests_dir.exists():
            print(f"ERROR: Directory at '{smoke_tests_dir}' does not exist.")
            sys.exit(1)

        setup_env()

        # CRITICAL: Query unique devices for iteration.
        # HIP_VISIBLE_DEVICES will be set inside set_gpu_execution_policy.
        unique_devices = get_unique_supported_devices(args.amdgpu_family)

        print(f"Will run smoke tests on {len(unique_devices)} unique device(s)")

        # Track overall success
        overall_retcode = 0

        pytest_args = [
            f"{smoke_tests_dir}",
        ]

        # Append any passthrough pytest args passed after "--"
        pytest_args.extend(passthrough_pytest_args)

        # Run smoke tests for each unique device iteratively using offset
        for offset in range(len(unique_devices)):
            # Set HIP_VISIBLE_DEVICES for this specific device using "single" policy
            ((arch, device_idx),) = set_gpu_execution_policy(
                unique_devices, policy="single", offset=offset
            )

            print(f"\n{'='*60}")
            print(f"Running smoke tests on device {device_idx} ({arch})")
            print(f"{'='*60}")

            # Use a subprocess so PyTorch re-reads HIP_VISIBLE_DEVICES per device.
            result = subprocess.run(
                [sys.executable, "-m", "pytest", *pytest_args],
                env=os.environ.copy(),
            )
            retcode = result.returncode
            print(
                f"Pytest finished for device {device_idx} ({arch}) with return code: {retcode}"
            )

            # Track if any test run failed
            if retcode != 0:
                overall_retcode = retcode

        print(f"\n{'='*60}")
        print(f"All smoke tests completed. Overall return code: {overall_retcode}")
        print(f"{'='*60}")

        return overall_retcode

    except (ValueError, IndexError) as e:
        print(f"[ERROR] Exception in PyTorch smoke-tests runner: {e}")
        sys.exit(1)


if __name__ == "__main__":
    # Lets make this script return pytest exit code (success or failure)
    sys.exit(main())