TheRock/rocm-systems/.github/scripts/therock_configure_ci.py

"""
This script determines which build flags and tests to run based on the
GitHub event type and configured subtrees/projects.

For push and pull_request events, SUBTREES is used to decide which parts
of the repository changed and which projects to run. Nightly (schedule)
and some workflow_dispatch invocations do not require SUBTREES.
"""

import fnmatch
import json
import logging
from pathlib import Path
import subprocess
import sys
from therock_matrix import (
    subtree_to_project_map,
    project_map,
    trigger_windows_ci_for_subtrees_paths,
    windows_only_subtrees,
)
import time
from typing import List, Mapping, Optional, Iterable
import os

# Valid test types in order of comprehensiveness (least to most)
VALID_TEST_TYPES = ["quick", "standard", "comprehensive", "full"]

logging.basicConfig(level=logging.INFO)


def set_github_output(d: Mapping[str, str]):
    """Sets GITHUB_OUTPUT values.
    See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs
    """
    logging.info(f"Setting github output:\n{d}")
    step_output_file = os.environ.get("GITHUB_OUTPUT", "")
    if not step_output_file:
        logging.warning(
            "Warning: GITHUB_OUTPUT env var not set, can't set github outputs"
        )
        return
    with open(step_output_file, "a") as f:
        f.writelines(f"{k}={v}" + "\n" for k, v in d.items())


def retry(max_attempts, delay_seconds, exceptions):
    def decorator(func):
        def newfn(*args, **kwargs):
            attempt = 0
            while attempt < max_attempts:
                try:
                    return func(*args, **kwargs)
                except exceptions as e:
                    print(
                        f"Exception {str(e)} thrown when attempting to run , attempt {attempt} of {max_attempts}"
                    )
                    attempt += 1
                    if attempt < max_attempts:
                        backoff = delay_seconds * (2 ** (attempt - 1))
                        time.sleep(backoff)
            return func(*args, **kwargs)

        return newfn

    return decorator


@retry(max_attempts=3, delay_seconds=2, exceptions=(TimeoutError))
def get_modified_paths(base_ref: str) -> Optional[Iterable[str]]:
    """Returns the paths of modified files relative to the base reference."""
    return subprocess.run(
        ["git", "diff", "--name-only", base_ref],
        stdout=subprocess.PIPE,
        check=True,
        text=True,
        timeout=60,
    ).stdout.splitlines()


GITHUB_WORKFLOWS_CI_PATTERNS = [
    "therock*",
]


def is_path_workflow_file_related_to_ci(path: str) -> bool:
    return (
        any(
            fnmatch.fnmatch(path, ".github/workflows/" + pattern)
            for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
        )
        or any(
            fnmatch.fnmatch(path, ".github/scripts/" + pattern)
            for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
        )
        or any(
            fnmatch.fnmatch(path, ".github/actions/" + pattern)
            for pattern in GITHUB_WORKFLOWS_CI_PATTERNS
        )
    )


def check_for_workflow_file_related_to_ci(paths: Optional[Iterable[str]]) -> bool:
    if paths is None:
        return False
    return any(is_path_workflow_file_related_to_ci(p) for p in paths)


def check_trigger_windows_ci_for_subtree_path(path):
    """Returns true if path matches any of matches windows ci subtree patterns"""
    for windows_ci_subtree_patterns in trigger_windows_ci_for_subtrees_paths:
        if fnmatch.fnmatch(path, windows_ci_subtree_patterns):
            return True
    return False


def check_trigger_windows_ci_for_subtree(subtree: str) -> bool:
    """Returns true if the subtree root corresponds to a Windows CI-triggering subtree.

    Used for workflow_dispatch where explicit subtrees are provided rather than
    modified file paths. Patterns like 'projects/clr/*' are matched by stripping
    the trailing glob to get the subtree prefix 'projects/clr'.
    """
    for pattern in trigger_windows_ci_for_subtrees_paths:
        subtree_prefix = pattern.rstrip("/*").rstrip("/")
        if subtree == subtree_prefix or subtree.startswith(subtree_prefix + "/"):
            return True
    return False


# Paths matching any of these patterns are considered to have no influence over
# build or test workflows so any related jobs can be skipped if all paths
# modified by a commit/PR match a pattern in this list.
SKIPPABLE_PATH_PATTERNS = [
    # Miscellaneous git/github files
    "*.gitignore",
    "*.pre-commit-config.*",
    ".github/label*.yml",
    "*CODEOWNERS",
    "*LICENSE",
    "*/.markdownlint-ci2.yaml",
    "tools/systems_pr_bot/*",
    # Documentation files
    "docs/*",
    "*.md",
    "*.rtf",
    "*.rst",
    "*/.readthedocs.yaml",
    "*/.spellcheck.local.yaml",
    "*/.wordlist.txt",
    "projects/*/docs/*",
    "shared/*/docs/*",
    # Changes to experimental code do not run standard build/test workflows.
    "experimental/*",
    # WSL support files (should these still be excluded?)
    "projects/rocr-runtime/libhsakmt/src/dxg/*",
]


def is_path_skippable(path: str) -> bool:
    """Determines if a given relative path to a file matches any skippable patterns."""
    # A workflow or action file only affects TheRock CI when it matches the
    # CI-related patterns. Treat other workflow and action files as skippable so
    # they don't need to be enumerated one by one.
    if path.startswith((".github/workflows/", ".github/actions/")):
        return not is_path_workflow_file_related_to_ci(path)
    return any(fnmatch.fnmatch(path, pattern) for pattern in SKIPPABLE_PATH_PATTERNS)


def check_for_non_skippable_path(paths: Optional[Iterable[str]]) -> bool:
    """Returns true if at least one path is not in the skippable set."""
    if paths is None:
        return False
    return any(not is_path_skippable(p) for p in paths)


def get_pr_labels(args) -> List[str]:
    """Gets a list of labels applied to a pull request."""
    data = json.loads(args.get("pr_labels", "{}"))
    labels = []
    for label in data.get("labels", []):
        labels.append(label["name"])
    return labels


def check_rccl_changes(modified_paths: Optional[Iterable[str]]) -> bool:
    """Returns true if any files under projects/rccl/ were modified."""
    if modified_paths is None:
        return False
    return any(path.startswith("projects/rccl/") for path in modified_paths)


def check_hip_rocr_changes(modified_paths: Optional[Iterable[str]]) -> bool:
    """Returns true if any HIP or ROCR files were modified (excluding docs).

    Also returns true if TheRock CI workflow files are modified.
    """
    if modified_paths is None:
        return False

    hip_rocr_prefixes = [
        "projects/clr/",
        "projects/hip/",
        "projects/hip-tests/",
        "projects/rocr-runtime/",
    ]

    # Patterns to ignore (docs, etc.)
    ignore_patterns = [
        "*.md",
        "*/docs/*",
        "*/.gitignore",
        "*/README*",
        "*/CONTRIBUTING*",
        "*/LICENSE*",
    ]

    def is_ignored(path: str) -> bool:
        return any(fnmatch.fnmatch(path, pattern) for pattern in ignore_patterns)

    def is_hip_rocr_code(path: str) -> bool:
        return any(path.startswith(prefix) for prefix in hip_rocr_prefixes)

    # Check for CI workflow changes
    if check_for_workflow_file_related_to_ci(modified_paths):
        return True

    # Check for HIP/ROCR code changes (excluding ignored files)
    return any(
        is_hip_rocr_code(path) and not is_ignored(path) for path in modified_paths
    )


def is_rccl_path(path: str) -> bool:
    """Returns true if path is under projects/rccl/."""
    return path.startswith("projects/rccl/")


def get_matched_subtree(path: str) -> Optional[str]:
    """Returns the subtree that matches the path, or None if no match."""
    for subtree in subtree_to_project_map:
        if path.startswith(subtree + "/") or path == subtree:
            return subtree
    return None


def check_only_rccl_changes(modified_paths: Iterable[str]) -> bool:
    """Returns true if all modified paths are either RCCL or match known subtrees."""
    for path in modified_paths:
        if not is_rccl_path(path) and get_matched_subtree(path) is None:
            return False
    return check_rccl_changes(modified_paths)


def retrieve_projects(args):
    # Default test type is "standard" for normal CI runs
    test_type = "standard"

    # Nightly (schedule): use same test coverage as TheRock submodule bump PRs —
    # single nightly job with THEROCK_ENABLE_ALL=ON and full projects_to_test list.
    # Run all builds and tests including RCCL.
    if args.get("is_nightly"):
        nightly_config = project_map.get("nightly")
        if not nightly_config:
            logging.warning(
                "No 'nightly' entry in project_map, nightly will have no jobs"
            )
            return [], "comprehensive"
        # Nightly runs use comprehensive testing
        test_type = "comprehensive"
        # Run full coverage on both Linux and Windows (no path-based skip).
        return [
            {
                "cmake_options": nightly_config.get("cmake_options", ""),
                "projects_to_test": nightly_config.get("projects_to_test", ""),
            }
        ], test_type

    # Check if CI should be skipped based on modified paths
    # (only for push and pull_request events, not workflow_dispatch or nightly)
    base_ref = args.get("base_ref")
    modified_paths = get_modified_paths(base_ref)
    print("modified_paths (max 200):", modified_paths[:200])

    # If only skippable paths were modified, skip CI
    if args.get("is_push") or args.get("is_pull_request"):
        if not check_for_non_skippable_path(modified_paths):
            logging.info("Only skippable paths were modified, skipping CI")
            return [], test_type

        # Check for ci:skip label on PRs
        if args.get("is_pull_request"):
            pr_labels = get_pr_labels(args)
            if "ci:skip" in pr_labels:
                logging.info("`ci:skip` label was added, skipping CI")
                return [], test_type

    # Push event → check which subtrees were modified
    if args.get("is_push"):
        matched_subtrees = {get_matched_subtree(p) for p in modified_paths} - {None}

        # Change in CI workflow triggers full subtree evaluation with quick tests
        if check_for_workflow_file_related_to_ci(modified_paths):
            logging.info(
                "CI workflow files changed, evaluating all subtrees with quick tests"
            )
            subtrees = list(subtree_to_project_map.keys())
            test_type = "quick"
        elif matched_subtrees:
            # Known subtrees changed - run CI for those subtrees
            subtrees = list(matched_subtrees)
        elif check_only_rccl_changes(modified_paths):
            # Only RCCL changes - skip regular CI, RCCL CI will handle it
            logging.info("Only RCCL changes detected on push, skipping regular CI")
            subtrees = []
        elif modified_paths:
            # Files changed but no known subtree matched (and not RCCL-only)
            logging.info(
                "Modified files did not match known subtrees, evaluating all projects"
            )
            subtrees = list(subtree_to_project_map.keys())
        else:
            subtrees = []

    # Manual workflow dispatch: respect explicit project selection, bypass CI file change detection
    elif args.get("is_workflow_dispatch"):
        if args.get("input_projects") == "all":
            subtrees = list(subtree_to_project_map.keys())
        else:
            subtrees = args.get("input_projects", "").split()

    else:
        # Determine which subtrees were modified (only needed for non-push/dispatch paths)
        matched_subtrees = set()
        for path in modified_paths:
            for subtree in subtree_to_project_map:
                if path.startswith(subtree + "/") or path == subtree:
                    matched_subtrees.add(subtree)

        # Change in CI workflow triggers full subtree evaluation with quick tests
        if check_for_workflow_file_related_to_ci(modified_paths):
            logging.info(
                "CI workflow files changed, evaluating all subtrees with quick tests"
            )
            subtrees = list(subtree_to_project_map.keys())
            test_type = "quick"

        # Pull request
        elif args.get("is_pull_request"):
            if args.get("input_subtrees"):
                subtrees = args.get("input_subtrees").split()
            else:
                subtrees = list(matched_subtrees)

        # Default case
        else:
            subtrees = list(matched_subtrees)

        # If files changed but no subtree matched → evaluate all
        if modified_paths and not subtrees and not check_rccl_changes(modified_paths):
            logging.info(
                "Modified files did not match known subtrees, evaluating all projects"
            )
            subtrees = list(subtree_to_project_map.keys())

    # Holds the python-specific cmake options passed to TheRock build.
    common_python_options = []

    # Linux CI skip logic: exclude Windows-only subtrees so they don't
    # produce Linux projects. If nothing remains, Linux CI is skipped.
    if args.get("platform") == "linux":
        subtrees = [s for s in subtrees if s not in windows_only_subtrees]

        # Common Python executable options for all builds.
        # Replaces TheRock's manylinux build behavior.
        # See build_tools/github_actions/manylinux_config.py in TheRock.
        common_python_options = [
            "-DTHEROCK_SHARED_PYTHON_EXECUTABLES=/opt/python-shared/cp310-cp310/bin/python3;/opt/python-shared/cp311-cp311/bin/python3;/opt/python-shared/cp312-cp312/bin/python3;/opt/python-shared/cp313-cp313/bin/python3;/opt/python-shared/cp314-cp314/bin/python3",
            "-DTHEROCK_DIST_PYTHON_EXECUTABLES=/opt/python/cp310-cp310/bin/python;/opt/python/cp311-cp311/bin/python;/opt/python/cp312-cp312/bin/python;/opt/python/cp313-cp313/bin/python",
        ]

    # Windows CI skip logic: skip if neither the modified file paths nor the
    # explicitly selected subtrees require Windows CI.
    if args.get("platform") == "windows":
        if args.get("is_workflow_dispatch"):
            if not any(check_trigger_windows_ci_for_subtree(s) for s in subtrees):
                logging.info("Selected subtrees do not require Windows CI, skipping")
                return [], test_type
        elif not any(
            check_trigger_windows_ci_for_subtree_path(path) for path in modified_paths
        ):
            logging.info("Modified paths do not require Windows CI, skipping")
            return [], test_type
    # Determine logical projects impacted
    projects = {
        subtree_to_project_map[subtree]
        for subtree in subtrees
        if subtree in subtree_to_project_map
    }

    if not projects:
        return [], test_type

    merged_flags = set()
    merged_tests = set()
    enable_all = False

    for project in projects:
        config = project_map.get(project)
        if not config:
            continue
        cmake_options = config.get("cmake_options", [])
        # Handle both array and string formats for backwards compatibility
        if isinstance(cmake_options, str):
            flags = [f.strip() for f in cmake_options.split()]
        else:
            flags = cmake_options
        if "-DTHEROCK_ENABLE_ALL=ON" in flags:
            enable_all = True
        merged_flags.update(flags)
        tests = config.get("projects_to_test", "")
        if tests:
            merged_tests.update(t.strip() for t in tests.split(","))
    if enable_all:
        final_flags_list = ["-DTHEROCK_ENABLE_ALL=ON"]
    else:
        final_flags_list = sorted(merged_flags)
    # Always append -DTHEROCK_ENABLE_CORE=ON as a default at the end
    final_flags_list.append("-DTHEROCK_ENABLE_CORE=ON")
    # Always append the Python options.
    final_flags_list += common_python_options
    # Removing duplicates
    final_flags_list = list(set(final_flags_list))
    final_flags = " ".join(final_flags_list)

    return [
        {
            "cmake_options": final_flags,
            "projects_to_test": ", ".join(sorted(merged_tests)),
        }
    ], test_type


def select_build_runner(platform: str) -> str:
    """Select a build runner label based on platform and build variant."""
    # TheRock is checked out alongside this repository in therock-ci.yml, but it
    # is not available in a standalone rocm-systems checkout. Keep this import
    # local so the rest of this module, including its unit tests, can run without
    # TheRock.
    therock_actions_path = Path("TheRock") / "build_tools" / "github_actions"
    sys.path.insert(0, str(therock_actions_path))
    from amdgpu_family_matrix import get_build_runner_labels, select_weighted_label

    build_runner_labels = get_build_runner_labels()
    if platform not in build_runner_labels:
        # Platform not configured for weighted selection, return default
        print(f"  No build runner config for platform {platform}, using default")
        return ""

    platform_config = build_runner_labels[platform]

    labels_config = platform_config["default"]
    context_name = f"build-runner ({platform})"

    return select_weighted_label(labels_config, context_name)


def run(args):
    platform = args.get("platform")
    project_to_run, test_type = retrieve_projects(args)
    build_runs_on = select_build_runner(platform)
    outputs = {
        "projects": json.dumps(project_to_run),
        "build_runs_on": build_runs_on,
        "test_type": test_type,
    }

    # Determine if RCCL CI should run (only relevant for Linux platform)
    if args.get("platform") == "linux":
        if args.get("is_nightly"):
            # Nightly runs always run RCCL CI
            outputs["run_linux_rccl_ci"] = "true"
        elif args.get("is_workflow_dispatch"):
            # For workflow_dispatch, check if projects/rccl was explicitly selected
            input_projects = args.get("input_projects", "")
            if "projects/rccl" in input_projects:
                outputs["run_linux_rccl_ci"] = "true"
            else:
                outputs["run_linux_rccl_ci"] = "false"
        else:
            # For push/PR events, check if any files under projects/rccl/ changed
            base_ref = args.get("base_ref")
            modified_paths = get_modified_paths(base_ref)
            if check_rccl_changes(modified_paths):
                outputs["run_linux_rccl_ci"] = "true"
            else:
                outputs["run_linux_rccl_ci"] = "false"

    # Determine if MI455 CI should run (only for PRs on Linux when HIP/ROCR or CI changes)
    if args.get("platform") == "linux":
        if args.get("is_pull_request"):
            base_ref = args.get("base_ref")
            modified_paths = get_modified_paths(base_ref)
            if check_for_workflow_file_related_to_ci(
                modified_paths
            ) or check_hip_rocr_changes(modified_paths):
                outputs["run_mi455_test"] = "true"
            else:
                outputs["run_mi455_test"] = "false"
        else:
            # MI455 CI only runs on PRs, not push/nightly/workflow_dispatch
            outputs["run_mi455_test"] = "false"

    set_github_output(outputs)
    return outputs


if __name__ == "__main__":
    args = {}
    github_event_name = os.getenv("GITHUB_EVENT_NAME")
    github_workflow = os.getenv("GITHUB_WORKFLOW", "")
    args["is_pull_request"] = github_event_name == "pull_request"
    args["is_push"] = github_event_name == "push"
    args["is_workflow_dispatch"] = github_event_name == "workflow_dispatch"
    # Nightly: either scheduled run or manual dispatch of the nightly workflow
    args["is_nightly"] = github_event_name == "schedule" or (
        github_event_name == "workflow_dispatch"
        and github_workflow == "TheRock CI Nightly"
    )

    input_subtrees = os.getenv("SUBTREES", "")
    args["input_subtrees"] = input_subtrees

    input_projects = os.getenv("PROJECTS", "")
    args["input_projects"] = input_projects

    args["pr_labels"] = os.environ.get("PR_LABELS", '{"labels": []}')

    input_platform = os.getenv("PLATFORM")
    args["platform"] = input_platform

    args["base_ref"] = os.environ.get("BASE_REF", "HEAD^")

    logging.info(f"Retrieved arguments {args}")

    run(args)