TheRock/build_tools/github_actions/detect_external_repo_config.py

#!/usr/bin/env python3
# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

"""
Detects external repository configuration for TheRock CI workflows.

This script determines build configuration settings based on the external repository
being built (rocm-libraries, rocm-systems, etc.). It outputs GitHub Actions variables
that control checkout steps, patches, and build options.

Usage:
    python detect_external_repo_config.py --repository <repository_name>

Examples:
    # Config for rocm-libraries:
    python build_tools/github_actions/detect_external_repo_config.py --repository rocm-libraries

    # Config for rocm-systems:
    python build_tools/github_actions/detect_external_repo_config.py --repository rocm-systems

    # Include a workspace path to produce an extra_cmake_options entry:
    python build_tools/github_actions/detect_external_repo_config.py --repository rocm-libraries --workspace "$GITHUB_WORKSPACE/source-repo"

Output (GitHub Actions format):
    cmake_source_var=THEROCK_ROCM_LIBRARIES_SOURCE_DIR
    submodule_path=rocm-libraries
    fetch_sources_args=--skip-submodules rocm-libraries
"""

import argparse
import importlib.util
import json
import os
import sys
from functools import lru_cache
from pathlib import Path
from typing import Dict, Any, Optional

from github_actions_api import gha_set_output


# Repository configuration map
REPO_CONFIGS: Dict[str, Dict[str, Any]] = {
    "rocm-libraries": {
        "cmake_source_var": "THEROCK_ROCM_LIBRARIES_SOURCE_DIR",
        "submodule_path": "rocm-libraries",
        "skip_submodules": ["rocm-libraries"],
    },
    "rocm-systems": {
        "cmake_source_var": "THEROCK_ROCM_SYSTEMS_SOURCE_DIR",
        "submodule_path": "rocm-systems",
        "skip_submodules": ["rocm-systems"],
        "dvc_projects": ["external-rocm-systems"],
    },
    "rocgdb": {
        "cmake_source_var": "THEROCK_ROCGDB_SOURCE_DIR",
        "submodule_path": "debug-tools/rocgdb/source",
        "skip_submodules": ["rocgdb"],
    },
    # Future repos can be added here:
    # "llvm-project": {...},
    # "hipify": {...},
    # "libhipcxx": {...},
}


def _log_warning(message: str) -> None:
    """Helper to log warning messages to stderr."""
    print(f"WARNING: {message}", file=sys.stderr)


def get_repo_config(repo_name: str) -> Dict[str, Any]:
    """Returns config for a known external repo name."""
    if repo_name not in REPO_CONFIGS:
        raise ValueError(
            f"Unknown external repository: {repo_name}\n"
            f"Known repositories: {', '.join(REPO_CONFIGS.keys())}"
        )

    return REPO_CONFIGS[repo_name]


@lru_cache()
def get_external_repo_path(repo_name: str) -> Path:
    """Determines the path to the external repository checkout.

    This function is cached (using functools.lru_cache) to avoid repeated
    filesystem lookups. The path for a given repo_name is computed once
    and reused for all subsequent calls during program execution.

    This function encapsulates the logic for finding where an external repo
    is checked out in different scenarios (external repo calling TheRock,
    test integration workflows, TheRock CI, etc.).

    Args:
        repo_name (str): Repository name (e.g., "rocm-libraries", "rocm-systems")

    Returns:
        Path: Path to the external repository root directory (cached after first call)

    Raises:
        ValueError: If the external repo path cannot be determined
    """
    try:
        repo_config = get_repo_config(repo_name)
    except ValueError as e:
        raise ValueError(f"Unknown repository: {repo_name}") from e

    # Priority order for determining external repo location:

    # 1. EXTERNAL_SOURCE_PATH environment variable
    #    Set in test integration workflows where TheRock is main checkout
    external_source_env = os.environ.get("EXTERNAL_SOURCE_PATH")
    if external_source_env:
        workspace_env = os.environ.get("GITHUB_WORKSPACE")
        if workspace_env:
            base_path = Path(workspace_env)
        else:
            base_path = Path.cwd()
            _log_warning(
                "EXTERNAL_SOURCE_PATH set but GITHUB_WORKSPACE not set, using CWD as base"
            )
        repo_path = base_path / external_source_env
        # Validate that the path ends with the expected repo name
        if (
            repo_path.exists()
            and _is_valid_repo_path(repo_path)
            and repo_path.name == repo_name
        ):
            print(
                f"Found external repo via EXTERNAL_SOURCE_PATH: {repo_path}",
                file=sys.stderr,
            )
            return repo_path

    # 2. Current directory (external repo calling TheRock CI)
    #    Most common case when external repos use TheRock workflows
    if _is_valid_repo_path(Path.cwd()):
        print(f"Found external repo at CWD: {Path.cwd()}", file=sys.stderr)
        return Path.cwd()

    raise ValueError(
        f"Could not find external repo '{repo_name}'. Checked:\n"
        f"  - EXTERNAL_SOURCE_PATH: {external_source_env or 'not set'}\n"
        f"  - CWD: {Path.cwd()}"
    )


def _is_valid_repo_path(path: Path) -> bool:
    """Validate that a path is a git repository with required TheRock integration scripts.

    External repositories that integrate with TheRock must have:
    - A .github/scripts/ directory
    - therock_matrix.py: Defines project build matrix and test lists
    - therock_configure_ci.py: CI configuration including skippable path patterns

    Args:
        path: Path to check

    Returns:
        True if path is a valid external repo with all required integration scripts
    """
    # Check for git repository (can be file or directory for worktrees)
    git_path = path / ".git"
    if not git_path.exists():
        return False

    # Check for .github/scripts directory (external repo structure)
    scripts_dir = path / ".github" / "scripts"
    if not scripts_dir.exists() or not scripts_dir.is_dir():
        return False

    # Check for required TheRock integration scripts
    required_scripts = ["therock_matrix.py", "therock_configure_ci.py"]
    for script_name in required_scripts:
        script_path = scripts_dir / script_name
        if not script_path.exists():
            return False

    return True


def import_external_repo_module(repo_name: str, module_name: str) -> Optional[Any]:
    """Dynamically import a module from an external repo's .github/scripts directory.

    Args:
        repo_name (str): Repository name (e.g., "rocm-libraries", "rocm-systems")
        module_name (str): Module name without .py extension (e.g., "therock_matrix")

    Returns:
        Optional[Any]: The imported module, or None if import fails

    Raises:
        ValueError: If the external repo path cannot be determined
    """
    # Get the validated repo path (will raise ValueError if invalid)
    repo_path = get_external_repo_path(repo_name)

    # All external repos follow the same convention: .github/scripts/
    script_path = repo_path / ".github" / "scripts" / f"{module_name}.py"

    if not script_path.exists():
        _log_warning(f"Could not find {module_name}.py at {script_path}")
        return None

    print(f"Importing {module_name} from: {script_path}", file=sys.stderr)

    try:
        spec = importlib.util.spec_from_file_location(
            f"{repo_name}.{module_name}", script_path
        )
        if spec and spec.loader:
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            return module
        else:
            _log_warning(
                f"Could not create module spec for {module_name} at {script_path}"
            )
            return None
    except ImportError as e:
        _log_warning(f"Failed to import {module_name} from {repo_name}: {e}")
        return None
    except Exception as e:
        print(
            f"ERROR: Unexpected error importing {module_name} from {repo_name}: {type(e).__name__}: {e}",
            file=sys.stderr,
        )
        raise


def get_skip_patterns(repo_name: str) -> list[str]:
    """Get skip patterns from external repo's therock_configure_ci.py.

    These are file path patterns that, when ALL modified files in a PR match at least
    one pattern, indicate the changes have no impact on CI workflows. When this occurs,
    TheRock CI will skip build/test jobs to save resources.

    Example patterns: ["*.md", "docs/*", ".github/workflows/*"]

    See: https://github.com/ROCm/rocm-libraries/blob/develop/.github/scripts/therock_configure_ci.py

    Args:
        repo_name (str): Repository name (e.g., "rocm-libraries", "rocm-systems")

    Returns:
        list[str]: List of skip patterns, or empty list if not found
    """
    configure_module = import_external_repo_module(repo_name, "therock_configure_ci")
    if configure_module and hasattr(configure_module, "SKIPPABLE_PATH_PATTERNS"):
        patterns = configure_module.SKIPPABLE_PATH_PATTERNS
        print(
            f"Loaded {len(patterns)} skip patterns from {repo_name}",
            file=sys.stderr,
        )
        return patterns
    return []


def get_test_list(repo_name: str) -> list[str]:
    """Get test list from external repo's therock_matrix.py project_map.

    Args:
        repo_name (str): Repository name (e.g., "rocm-libraries", "rocm-systems")

    Returns:
        list[str]: List of test names, or empty list if not found
    """
    matrix_module = import_external_repo_module(repo_name, "therock_matrix")
    if not matrix_module or not hasattr(matrix_module, "project_map"):
        return []

    # Collect all unique tests from all projects
    # NOTE: We ignore their cmake_options since we're doing full builds
    all_tests = set()
    project_map = matrix_module.project_map

    for project_config in project_map.values():
        tests = project_config.get("project_to_test", [])
        # Handle both list and comma-separated string formats
        if isinstance(tests, str):
            tests = [t.strip() for t in tests.split(",")]
        all_tests.update(tests)

    if all_tests:
        test_list = sorted(all_tests)
        print(f"Loaded {len(test_list)} tests from {repo_name}", file=sys.stderr)
        return test_list

    return []


def output_github_actions_vars(config: Dict[str, Any]) -> None:
    """Writes config as GitHub Actions outputs using the standard utility.

    Args:
        config: Configuration dictionary with keys like 'cmake_source_var',
            'submodule_path', etc. Values should be strings or booleans.

    Note:
        Uses gha_set_output() from github_actions_api.py which handles
        writing to GITHUB_OUTPUT file or stdout for local testing.
        Booleans are converted to lowercase strings for bash compatibility.
    """
    # Convert booleans to lowercase strings for bash compatibility
    normalized_config = {}
    for key, value in config.items():
        if isinstance(value, bool):
            normalized_config[key] = str(value).lower()
        else:
            normalized_config[key] = str(value)

    gha_set_output(normalized_config)


def main(argv=None):
    """Main entry point for the script.

    Args:
        argv: Command line arguments (defaults to sys.argv if None)

    Returns:
        Exit code (0 for success, 1 for error)
    """
    parser = argparse.ArgumentParser(
        description=(
            "Detect external repository configuration for TheRock CI workflows.\n\n"
            "This script determines build configuration settings based on the external\n"
            "repository being built (rocm-libraries, rocm-systems, etc.). It outputs\n"
            "GitHub Actions variables that control checkout steps, patches, and build options.\n\n"
            "Output Format (GitHub Actions):\n"
            "  cmake_source_var=THEROCK_ROCM_LIBRARIES_SOURCE_DIR\n"
            "  submodule_path=rocm-libraries\n"
            "  fetch_sources_args=--skip-submodules rocm-libraries"
        ),
        epilog=(
            "Examples:\n"
            "  # Config for rocm-libraries:\n"
            "  python build_tools/github_actions/detect_external_repo_config.py \\\n"
            "    --repository rocm-libraries\n\n"
            "  # Config for rocm-systems:\n"
            "  python build_tools/github_actions/detect_external_repo_config.py \\\n"
            "    --repository rocm-systems\n\n"
            "  # Include workspace path for CMake options:\n"
            "  python build_tools/github_actions/detect_external_repo_config.py \\\n"
            '    --repository rocm-libraries --workspace "$GITHUB_WORKSPACE/source-repo"\n\n'
            "  # List all known repositories:\n"
            "  python build_tools/github_actions/detect_external_repo_config.py --list"
        ),
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--external-repo-json",
        type=str,
        help="JSON object with 'repository' (e.g., 'ROCm/rocm-libraries') and 'ref' fields.",
    )
    parser.add_argument(
        "--repository",
        help="Repository name (e.g., rocm-libraries, rocm-systems). "
        "Alternative to --external-repo-json for direct invocation.",
    )
    parser.add_argument(
        "--list",
        action="store_true",
        help="List all known repository configurations",
    )

    args = parser.parse_args(argv)

    if args.list:
        print("Known external repositories:")
        for repo_name in REPO_CONFIGS.keys():
            print(f"  - {repo_name}")
        return 0

    # Parse external repo JSON if provided, extract repository name
    source_repository = None
    source_ref = ""
    if args.external_repo_json:
        try:
            external_repo = json.loads(args.external_repo_json)
            source_repository = external_repo.get("repository", "")
            source_ref = external_repo.get("ref", "")
            # Extract repo name from full name (e.g., "rocm-libraries" from "ROCm/rocm-libraries").
            # Lowercase so REPO_CONFIGS lookup is case-insensitive (e.g. "ROCgdb" -> "rocgdb").
            if "/" in source_repository:
                repo_name = source_repository.split("/")[-1].lower()
            else:
                repo_name = source_repository.lower()
            args.repository = repo_name
            print(
                f"Parsed external_repo: repository={source_repository}, ref={source_ref}",
                file=sys.stderr,
            )
        except json.JSONDecodeError as e:
            print(f"ERROR: Invalid JSON in --external-repo-json: {e}", file=sys.stderr)
            return 1

    if not args.repository:
        print(
            "ERROR: --repository or --external-repo-json is required",
            file=sys.stderr,
        )
        return 1

    try:
        config = get_repo_config(args.repository)

        # Log to stderr for visibility in CI logs
        print(f"Detected repository: {args.repository}", file=sys.stderr)
        print(f"Configuration: {config}", file=sys.stderr)

        # checkout_path is relative (for actions/checkout path: parameter)
        # Use "external-" prefix to avoid collisions with submodule paths
        checkout_path = f"external-{args.repository}"

        # Generate fetch_sources_args from skip_submodules and dvc_projects
        fetch_args_parts = []
        if "skip_submodules" in config and config["skip_submodules"]:
            skip_args = " ".join(config["skip_submodules"])
            fetch_args_parts.append(f"--skip-submodules {skip_args}")
        if "dvc_projects" in config and config["dvc_projects"]:
            dvc_args = " ".join(config["dvc_projects"])
            fetch_args_parts.append(f"--dvc-projects {dvc_args}")

        if fetch_args_parts:
            config["fetch_sources_args"] = " ".join(fetch_args_parts)
            print(
                f"Generated fetch_sources_args: {config['fetch_sources_args']}",
                file=sys.stderr,
            )

        # Build config_json with all fields needed by workflows
        final_source_repo = source_repository or f"ROCm/{args.repository}"
        source_package = (
            config["cmake_source_var"]
            .replace("THEROCK_", "")
            .replace("_SOURCE_DIR", "")
        )

        # Extract caller-supplied fields from external_repo JSON if provided
        projects = ""
        family_overrides = {}
        extra_cmake_options = ""

        if args.external_repo_json:
            try:
                external_repo = json.loads(args.external_repo_json)
                projects = external_repo.get("projects", "")

                # family_overrides allows external repos to specify per-family config
                # (e.g., test runners, test_labels_for_family) for their CI runs only
                family_overrides = external_repo.get("family_overrides", {})

                extra_cmake_options = external_repo.get("extra_cmake_options", "")
            except json.JSONDecodeError as e:
                print(
                    f"Warning: failed to parse external_repo_json: {e}",
                    file=sys.stderr,
                )

        config_json = {
            "repository": final_source_repo,
            "ref": source_ref,
            "checkout_path": checkout_path,
            "source_package": source_package,
            "fetch_sources_args": config.get("fetch_sources_args", ""),
            "extra_cmake_options": extra_cmake_options,
            "projects": projects,
            "family_overrides": family_overrides,
        }
        config["config_json"] = json.dumps(config_json)
        print(
            f"Generated config_json:\n{json.dumps(config_json, indent=2)}",
            file=sys.stderr,
        )

        output_github_actions_vars(config)
        return 0

    except ValueError as e:
        print(f"ERROR: {e}", file=sys.stderr)
        return 1


if __name__ == "__main__":
    sys.exit(main())