TheRock/build_tools/github_actions/reproduce_test_failure.py

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

"""
Reproduces a test failure from CI.

Usage:
    # Linux (uses Docker)
    python reproduce_test_failure.py --run-id 12345678 --repository ROCm/TheRock \
        --amdgpu-family gfx94X --test-script "python test.py" --platform linux

    # Windows (bare metal, requires admin PowerShell)
    python reproduce_test_failure.py --run-id 12345678 --repository ROCm/TheRock \
        --amdgpu-family gfx110X --test-script "python test.py" --platform windows

    # Setup only (drops into shell)
    python reproduce_test_failure.py --run-id 12345678 --repository ROCm/TheRock \
        --amdgpu-family gfx94X --test-script "python test.py" --setup-only
"""

import argparse
import platform
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

DEFAULT_CONTAINER_IMAGE = "ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest"

is_windows = platform.system() == "Windows"


def check_windows_package_installed(package: str) -> bool:
    """Check if a Windows package is already installed."""
    if platform.system() != "Windows":
        return False

    # Map package names to their executables or check commands
    checks = {
        "chocolatey": ["choco", "--version"],
        "git": ["git", "--version"],
        "python": ["python", "--version"],
        "cmake": ["cmake", "--version"],
        "ninja": ["ninja", "--version"],
        "ccache": ["ccache", "--version"],
        "uv": ["uv", "--version"],
    }

    if package not in checks:
        return False

    try:
        result = subprocess.run(
            checks[package],
            capture_output=True,
            timeout=5,
        )
        return result.returncode == 0
    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
        return False


def check_docker() -> bool:
    if not shutil.which("docker"):
        return False
    result = subprocess.run(["docker", "info"], capture_output=True)
    return result.returncode == 0


def build_reproduction_command(args: argparse.Namespace) -> str:
    """Build the command string for reproduction."""
    cmd = (
        f"python build_tools/github_actions/reproduce_test_failure.py "
        f"--run-id {args.run_id} "
        f"--repository {args.repository} "
        f"--amdgpu-family {args.amdgpu_family} "
        f'--test-script "{args.test_script}" '
    )
    if args.amdgpu_targets:
        cmd += f" --amdgpu-targets {args.amdgpu_targets}"
    if args.shard_index != "1":
        cmd += f" --shard-index {args.shard_index}"
    if args.total_shards != "1":
        cmd += f" --total-shards {args.total_shards}"
    if args.test_type != "full":
        cmd += f" --test-type {args.test_type}"
    if args.fetch_artifact_args:
        cmd += f' --fetch-artifact-args="{args.fetch_artifact_args}"'
    return cmd


def run_linux(args: argparse.Namespace) -> int:
    """Run reproduction in Docker for Linux."""
    if not check_docker():
        print("ERROR: Docker is not available. Install Docker and try again.")
        return 1

    fetch_cmd = (
        f"GITHUB_REPOSITORY={args.repository} "
        f"python build_tools/install_rocm_from_artifacts.py "
        f"--run-id {args.run_id} "
        f"--amdgpu-family {args.amdgpu_family}"
    )
    if args.amdgpu_targets:
        fetch_cmd += f" --amdgpu-targets {args.amdgpu_targets}"
    if args.fetch_artifact_args:
        fetch_cmd += f" {args.fetch_artifact_args}"

    steps = [
        (
            "Installing uv",
            "curl -LsSf https://astral.sh/uv/install.sh | bash && source $HOME/.local/bin/env",
        ),
        (
            "Cloning TheRock",
            "git clone https://github.com/ROCm/TheRock.git && cd TheRock",
        ),
        ("Creating virtual environment", "uv venv .venv && source .venv/bin/activate"),
        ("Installing dependencies", "uv pip install -r requirements-test.txt"),
        ("Downloading artifacts", fetch_cmd),
        (
            "Setting environment variables",
            " && ".join(
                [
                    "export THEROCK_BIN_DIR=./therock-build/bin",
                    "export OUTPUT_ARTIFACTS_DIR=./therock-build",
                    f"export SHARD_INDEX={args.shard_index}",
                    f"export TOTAL_SHARDS={args.total_shards}",
                    f"export TEST_TYPE={args.test_type}",
                ]
            ),
        ),
    ]

    if args.setup_only:
        steps.append(("Setup complete", f"echo 'Run: {args.test_script}'"))
    else:
        steps.append(
            (
                "Running test",
                f"{args.test_script} || echo 'Test failed with exit code '$?",
            )
        )

    total = len(steps)
    lines = ["set -e"]
    for i, (desc, cmd) in enumerate(steps, 1):
        lines.append(f"echo '[{i}/{total}] {desc}'")
        if i == total:
            lines.append("set +e")
        lines.append(cmd)
    lines.append("exec /bin/bash")

    docker_cmd = [
        "docker",
        "run",
        "--rm",
        "-it",
        "--ipc",
        "host",
        "--group-add",
        "video",
        "--device",
        "/dev/kfd",
        "--device",
        "/dev/dri",
        args.container_image,
        "/bin/bash",
        "-c",
        "\n".join(lines),
    ]

    try:
        return subprocess.run(docker_cmd).returncode
    except KeyboardInterrupt:
        return 130


def run_windows(args: argparse.Namespace) -> int:
    """Run reproduction on bare metal for Windows."""
    all_packages = ["chocolatey", "git", "python", "cmake", "ninja", "ccache", "uv"]

    # Check which packages are already installed
    installed = []
    missing = []
    for pkg in all_packages:
        if check_windows_package_installed(pkg):
            installed.append(pkg)
        else:
            missing.append(pkg)

    print("=" * 60)
    print("WINDOWS REPRODUCTION (bare metal)")
    print("=" * 60)
    print()

    if installed:
        print("Already installed:")
        for pkg in installed:
            print(f"  - {pkg}")
        print()

    # Only prompt if there are packages to install
    if missing:
        print("This will install the following packages on your system:")
        for pkg in missing:
            print(f"  - {pkg}")
        print()
        response = input("Continue? [y/N] ").strip().lower()
        if response not in ("y", "yes"):
            print("Aborted.")
            return 1
        print()
    else:
        print("All required packages are already installed.")
        print()

    fetch_cmd = (
        f"$env:GITHUB_REPOSITORY='{args.repository}'; "
        f"python build_tools/install_rocm_from_artifacts.py "
        f"--run-id {args.run_id} "
        f"--amdgpu-family {args.amdgpu_family}"
    )
    if args.amdgpu_targets:
        fetch_cmd += f" --amdgpu-targets {args.amdgpu_targets}"
    if args.fetch_artifact_args:
        fetch_cmd += f" {args.fetch_artifact_args}"

    # Build steps conditionally based on what's already installed
    steps = []

    if "chocolatey" in missing:
        steps.append(
            (
                "Installing chocolatey",
                (
                    "Set-ExecutionPolicy Bypass -Scope Process -Force; "
                    "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; "
                    "iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))"
                ),
            )
        )
        # Must use Import-Module to refresh env since [System.Environment] can be stale
        steps.append(
            (
                "Refreshing environment",
                "Import-Module $env:ChocolateyInstall\\helpers\\chocolateyProfile.psm1; refreshenv",
            )
        )

    # Check which choco packages need installing
    choco_packages = [
        p for p in ["git", "python", "cmake", "ninja", "ccache"] if p in missing
    ]
    if choco_packages:
        steps.append(
            ("Installing dependencies", f"choco install -y {' '.join(choco_packages)}")
        )
        steps.append(
            (
                "Refreshing environment",
                "Import-Module $env:ChocolateyInstall\\helpers\\chocolateyProfile.psm1; refreshenv",
            )
        )

    if "uv" in missing:
        steps.append(("Installing uv", "irm https://astral.sh/uv/install.ps1 | iex"))
        # Refresh PATH to pick up uv installation
        steps.append(
            (
                "Refreshing environment for uv",
                "$env:Path = [System.Environment]::GetEnvironmentVariable('Path','Machine') + ';' + [System.Environment]::GetEnvironmentVariable('Path','User') + ';' + \"$env:USERPROFILE\\.local\\bin\"",
            )
        )

    steps.extend(
        [
            (
                "Creating virtual environment",
                "uv venv .venv; .venv\\Scripts\\Activate.ps1",
            ),
            (
                "Installing Python dependencies",
                "uv pip install -r requirements-test.txt",
            ),
            ("Downloading artifacts", fetch_cmd),
            (
                "Setting environment variables",
                "; ".join(
                    [
                        "$env:THEROCK_BIN_DIR='./therock-build/bin'",
                        "$env:OUTPUT_ARTIFACTS_DIR='./therock-build'",
                        f"$env:SHARD_INDEX='{args.shard_index}'",
                        f"$env:TOTAL_SHARDS='{args.total_shards}'",
                        f"$env:TEST_TYPE='{args.test_type}'",
                    ]
                ),
            ),
        ]
    )

    if args.setup_only:
        steps.append(("Setup complete", f"Write-Host 'Run: {args.test_script}'"))
    else:
        steps.append(("Running test", args.test_script))

    total = len(steps)
    lines = ["$ErrorActionPreference = 'Stop'"]
    for i, (desc, cmd) in enumerate(steps, 1):
        lines.append(f"Write-Host '[{i}/{total}] {desc}'")
        if i == total:
            lines.append("$ErrorActionPreference = 'Continue'")
        lines.append(cmd)

    script_content = "\n".join(lines)

    # Write to temp file and execute
    with tempfile.NamedTemporaryFile(mode="w", suffix=".ps1", delete=False) as f:
        f.write(script_content)
        script_path = f.name

    try:
        if platform.system() != "Windows":
            print("NOTE: Not running on Windows. Printing script instead:")
            print("-" * 60)
            print(script_content)
            print("-" * 60)
            print(f"\nScript saved to: {script_path}")
            print("Copy this script to your Windows machine and run:")
            print(
                f"  powershell -ExecutionPolicy Bypass -File {Path(script_path).name}"
            )
            return 0

        result = subprocess.run(
            ["powershell", "-ExecutionPolicy", "Bypass", "-File", script_path],
        )
        return result.returncode
    except KeyboardInterrupt:
        return 130
    finally:
        if platform.system() == "Windows":
            Path(script_path).unlink(missing_ok=True)


def main() -> int:
    parser = argparse.ArgumentParser(description="Reproduce a test failure from CI")
    parser.add_argument("--run-id", required=True, help="GitHub Actions run ID")
    parser.add_argument("--repository", required=True, help="GitHub repository")
    parser.add_argument("--amdgpu-family", required=True, help="AMDGPU family")
    parser.add_argument(
        "--amdgpu-targets",
        default="",
        help="Comma-separated gfx targets (e.g. gfx942). Needed to fetch "
        "per-target kpack-split shards; optional for monolithic runs.",
    )
    parser.add_argument("--test-script", required=True, help="Test script to run")
    parser.add_argument("--shard-index", default="1", help="Shard index")
    parser.add_argument("--total-shards", default="1", help="Total shards")
    parser.add_argument("--test-type", default="full", help="Test type")
    parser.add_argument(
        "--container-image",
        default=DEFAULT_CONTAINER_IMAGE,
        help="Docker image (Linux only)",
    )
    parser.add_argument(
        "--fetch-artifact-args", nargs="?", default="", help="Extra artifact args"
    )
    parser.add_argument(
        "--setup-only", action="store_true", help="Setup only, don't run test"
    )
    parser.add_argument(
        "--print-cmd", action="store_true", help="Print reproduction command"
    )

    args = parser.parse_args()

    if args.print_cmd:
        print("To reproduce this failure, run:")
        print("  git clone https://github.com/ROCm/TheRock.git && cd TheRock")
        print(f"  {build_reproduction_command(args)}")
        return 0

    if is_windows:
        return run_windows(args)
    else:
        return run_linux(args)


if __name__ == "__main__":
    sys.exit(main())