""" This script determines which build flag and tests to run based on SUBTREES Required environment variables: - SUBTREES """ import fnmatch import json import logging from pathlib import Path import sys from therock_matrix import subtree_to_project_map, collect_projects_to_run from typing import Optional, Iterable, List import os from pr_detect_changed_subtrees import get_valid_prefixes, find_matched_subtrees from config_loader import load_repo_config from ci_utils import get_modified_paths, matches_paths, set_github_output # Add TheRock's github_actions to path for shared utilities THEROCK_ACTIONS_PATH = Path("TheRock") / "build_tools" / "github_actions" sys.path.insert(0, str(THEROCK_ACTIONS_PATH)) from amdgpu_family_matrix import get_build_runner_labels, select_weighted_label logging.basicConfig(level=logging.INFO) SCRIPT_DIR = Path(__file__).resolve().parent # Paths matching any of these patterns are considered to have no influence over # build or test workflows so any related jobs can be skipped if all paths # modified by a commit/PR match a pattern in this list. SKIPPABLE_PATH_PATTERNS = [ "*.clinerules", "*.cursorrules", "*.mdc", "*.md", "*.rst", "*.rtf", "*/.markdownlint-ci2.yaml", "*/.readthedocs.yaml", "*/.spellcheck.local.yaml", "*/.wordlist.txt", ".gitignore", "dnn-providers/*/.gitignore", "dnn-providers/*/docs/*", "docs/*", "projects/*/.gitignore", "projects/*/docs/*", # Tools are standalone scripts/utilities not part of the build or test pipeline. # Changes here should not trigger CI builds. "projects/hipdnn/tools/*", "shared/*/.gitignore", "shared/*/docs/*", "projects/composablekernel/Jenkinsfile", "projects/composablekernel/Docker*", "projects/composablekernel/client_example/*", "projects/composablekernel/codegen/*", "projects/composablekernel/dispatcher/*", "projects/composablekernel/example/*", "projects/composablekernel/experimental/*", "projects/composablekernel/profiler/*", "projects/composablekernel/python/*", "projects/composablekernel/rocm_ck/*", "projects/composablekernel/script/*", "projects/composablekernel/test/*", "projects/composablekernel/test_data/*", "projects/composablekernel/tile_engine/*", "projects/composablekernel/tutorial/*", "projects/composablekernel/vars/*", "projects/composablekernel/groovy/*", ] def is_path_skippable(path: str) -> bool: """Determines if a given relative path to a file matches any skippable patterns.""" return any(fnmatch.fnmatch(path, pattern) for pattern in SKIPPABLE_PATH_PATTERNS) def get_pr_labels(args) -> List[str]: """Gets a list of labels applied to a pull request.""" data = json.loads(args.get("pr_labels", "{}")) labels = [] for label in data.get("labels", []): labels.append(label["name"]) return labels def parse_test_labels(pr_labels: List[str]) -> tuple[List[str], Optional[str]]: """ Parse PR labels to extract test projects and test type. Examples: ['test:rocblas', 'test:miopen'] -> (['rocblas', 'miopen'], None) ['test:rocblas', 'test_type:comprehensive'] -> (['rocblas'], 'comprehensive') """ projects_to_test = [] test_type = None # Build label_to_project_map from subtree_to_project_map label_to_project_map = {} for subtree, project in subtree_to_project_map.items(): # Extract the project name from the subtree path (e.g., "projects/rocblas" -> "rocblas") if subtree.startswith("projects/") or subtree.startswith("dnn-providers/"): label_name = subtree.split("/")[-1] label_to_project_map[label_name] = project # Valid test types in order of comprehensiveness (least to most) valid_test_types = ["quick", "standard", "comprehensive", "full"] test_type_priority = {t: i for i, t in enumerate(valid_test_types)} for label in pr_labels: # Parse test:* labels for project selection if label.startswith("test:"): project_name = label[5:] # Remove 'test:' prefix if project_name in label_to_project_map: mapped_project = label_to_project_map[project_name] if mapped_project not in projects_to_test: projects_to_test.append(mapped_project) else: logging.warning(f"Unknown project in label: {label}") continue # Parse test_type:* labels elif label.startswith("test_type:"): label_test_type = label.split("test_type:")[ -1 ] # Remove 'test_type:' prefix if label_test_type in valid_test_types: # If multiple test_type labels, use the most comprehensive one if ( test_type is None or test_type_priority[label_test_type] > test_type_priority[test_type] ): test_type = label_test_type else: logging.warning(f"Unknown test type in label: {label}") return projects_to_test, test_type def check_for_non_skippable_path(paths: Optional[Iterable[str]]) -> bool: """Returns true if at least one path is not in the skippable set.""" if paths is None: return False return any(not is_path_skippable(p) for p in paths) THEROCK_CI_PATTERNS = [ ".github/workflows/therock*", ".github/scripts/therock*", ".github/actions/ci-env/action.yml", ] def check_for_workflow_file_related_to_ci(paths: Optional[Iterable[str]]) -> bool: if paths is None: return False return matches_paths(paths, THEROCK_CI_PATTERNS) def get_changed_path_projects(paths: Optional[Iterable[str]]) -> Iterable[str]: repo_config_path = Path(SCRIPT_DIR / ".." / "repos-config.json") config = load_repo_config(str(repo_config_path)) valid_prefixes = get_valid_prefixes(config) matched_subtrees = find_matched_subtrees(paths, valid_prefixes) return matched_subtrees def select_build_runner(platform: str) -> str: """Select a build runner label based on platform and build variant.""" build_runner_labels = get_build_runner_labels() if platform not in build_runner_labels: # Platform not configured for weighted selection, return default print(f" No build runner config for platform {platform}, using default") return "" platform_config = build_runner_labels[platform] labels_config = platform_config["default"] context_name = f"build-runner ({platform})" return select_weighted_label(labels_config, context_name) def retrieve_projects(args): # For pushes and pull_requests, we only want to test changed projects base_ref = args.get("base_ref") modified_paths = get_modified_paths(base_ref) # by default, we select standard tests test_type = "standard" # Variables to track if labels override defaults label_projects = [] label_test_type = None # Check if CI should be skipped based on modified paths # (only for push and pull_request events, not workflow_dispatch or nightly) if args.get("is_push") or args.get("is_pull_request"): paths_set = set(modified_paths) contains_non_skippable_files = check_for_non_skippable_path(paths_set) pr_labels = get_pr_labels(args) # Parse PR labels for test: and test_type: labels label_projects, label_test_type = parse_test_labels(pr_labels) # If test_type label is present, override the default test type if label_test_type: test_type = label_test_type logging.info(f"Test type overridden by label: {test_type}") # If only skippable paths were modified and no test labels, skip CI if not contains_non_skippable_files and not label_projects: logging.info("Only skippable paths were modified, skipping CI") return [], test_type if "skip-therockci" in pr_labels: logging.info("`skip-therockci` label was added, skipping CI") return [], test_type subtrees = get_changed_path_projects(modified_paths) # If test: labels are present (only for pull requests), add those projects to the build/test list if label_projects and args.get("is_pull_request"): logging.info(f"Projects specified by labels: {label_projects}") # Find at least one subtree that maps to each labeled project # We need actual subtrees because collect_projects_to_run expects them label_subtrees = [] for project in label_projects: # Find first subtree that maps to this project for subtree, mapped_project in subtree_to_project_map.items(): if mapped_project == project: label_subtrees.append(subtree) break # Only need one representative subtree per project # Combine file-based detection with label-based selection subtrees = list(set(subtrees + label_subtrees)) if args.get("is_workflow_dispatch"): if args.get("input_projects") == "all": subtrees = list(subtree_to_project_map.keys()) else: subtrees = args.get("input_projects").split() # If .github/*/therock* were changed for a push or pull request, run all subtrees if args.get("is_push") or args.get("is_pull_request"): related_to_therock_ci = check_for_workflow_file_related_to_ci(modified_paths) if related_to_therock_ci: logging.info( "Enabling all projects since a related workflow file was modified" ) subtrees = list(subtree_to_project_map.keys()) # Only override test_type if not already set by label (and it's a PR) if not (label_test_type and args.get("is_pull_request")): test_type = "quick" # for nightly runs, run everything with full tests if args.get("is_nightly"): subtrees = list(subtree_to_project_map.keys()) test_type = "comprehensive" project_to_run = collect_projects_to_run(subtrees) return project_to_run, test_type def run(args): platform = args.get("platform") project_to_run, test_type = retrieve_projects(args) build_runs_on = select_build_runner(platform) set_github_output( { f"{platform}_projects": json.dumps(project_to_run), "test_type": test_type, "build_runs_on": build_runs_on, } ) if __name__ == "__main__": args = {} github_event_name = os.getenv("GITHUB_EVENT_NAME") platform = os.getenv("PLATFORM") args["platform"] = platform args["is_pull_request"] = github_event_name == "pull_request" args["is_push"] = github_event_name == "push" args["is_workflow_dispatch"] = github_event_name == "workflow_dispatch" args["is_nightly"] = github_event_name == "schedule" args["pr_labels"] = os.environ.get("PR_LABELS", '{"labels": []}') input_projects = os.getenv("PROJECTS", "") args["input_projects"] = input_projects args["base_ref"] = os.environ.get("BASE_REF", "HEAD^") logging.info(f"Retrieved arguments {args}") run(args)