"""Generate manifest diff reports between two TheRock commits.
Compares submodule versions and generates HTML reports showing commit changes
for each component between builds.
Arguments:
--start Start commit SHA or workflow run ID (required unless using
--find-last-run or --pr-base-ref).
--end End commit SHA or workflow run ID (required).
--find-last-run Workflow filename (e.g., 'multi_arch_ci.yml'). When set,
--start is resolved as the head SHA of that workflow's
most recent run on --branch that concluded with
'success' or 'failure' (cancelled / skipped /
in-progress runs are ignored).
--pr-base-ref PR base branch name. When set, --start is resolved as
the merge-base between --end and the named branch via
the GitHub Compare API. Rebase-safe.
--workflow-mode Treat --start and --end as workflow run IDs instead of
commit SHAs.
--branch Branch to scope --find-last-run lookups against
(default: 'main'). Only consulted when --find-last-run
is set.
--output-dir Directory to write the HTML report into. If unset,
falls back to the TheRock root directory.
If no usable start ref can be derived, the script logs the reason and
exits 0 without writing a report.
Example usage:
python build_tools/generate_manifest_diff_report.py --start abc123 --end def456
python build_tools/generate_manifest_diff_report.py --end def456 --find-last-run multi_arch_ci.yml
python build_tools/generate_manifest_diff_report.py --end def456 --pr-base-ref main
python build_tools/generate_manifest_diff_report.py --start 12345 --end 67890 --workflow-mode
"""
# Standard library imports
import argparse
import html
import sys
import urllib.parse
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from urllib.error import HTTPError
# Establish script's location first (needed for local imports)
THIS_SCRIPT_DIR = Path(__file__).resolve().parent
THEROCK_DIR = THIS_SCRIPT_DIR.parent
sys.path.insert(0, str(THIS_SCRIPT_DIR))
# Local imports
from generate_therock_manifest import build_manifest_schema
from github_actions.github_actions_api import (
gha_append_step_summary,
gha_query_last_workflow_run,
gha_query_workflow_run_by_id,
gha_send_request,
)
# GitHub API constants
GITHUB_API_BASE = "https://api.github.com/repos"
ROCM_ORG = "ROCm"
THEROCK_REPO = "TheRock"
# Superrepos requiring component-level processing
SUPERREPO_NAMES = {"rocm-systems", "rocm-libraries"}
# Directories to scan within superrepos for components (In case new directories are added, we need to add them here)
SUPERREPO_COMPONENT_DIRS = ["shared", "projects"]
# Pagination constants for GitHub API commit fetching.
# MAX_PAGES * PER_PAGE = 2000 commits maximum.
# This limit prevents excessive API calls while covering most reasonable commit ranges.
# GitHub API returns commits in reverse chronological order, so we paginate until
# we find the start commit or hit the limit.
MAX_PAGES = 20 # Maximum number of API pages to fetch
PER_PAGE = 100 # Commits per page (GitHub API maximum is 100)
# Report identifiers
UNASSIGNED_KEY = "Unassigned"
# File paths
HTML_TEMPLATE_PATH = THIS_SCRIPT_DIR / "manifest_diff_report_template.html"
HTML_REPORT_PATH = THEROCK_DIR / "TheRockReport.html"
# =============================================================================
# Domain Classes
# =============================================================================
@dataclass
class Component:
"""A component within a superrepo (projects/rocblas, shared/hipblaslt)."""
path: str
name: str
status: str = "unchanged" # added, removed, unchanged, changed
commits: list[dict] = field(default_factory=list)
@dataclass
class Submodule:
"""A submodule in TheRock manifest."""
name: str
sha: str
api_base: str
branch: str
status: str = "unchanged" # removed, added, unchanged, changed, reverted
commits: list[dict] = field(default_factory=list)
start_sha: str = ""
end_sha: str = ""
@dataclass
class Superrepo(Submodule):
"""A superrepo submodule containing components (rocm-libraries, rocm-systems)."""
components: dict[str, Component] = field(default_factory=dict)
all_commits: list[dict] = field(default_factory=list)
commit_allocation: dict[str, list[dict]] = field(default_factory=dict)
@property
def added_components(self) -> list[str]:
return [c.name for c in self.components.values() if c.status == "added"]
@property
def removed_components(self) -> list[str]:
return [c.name for c in self.components.values() if c.status == "removed"]
@property
def changed_components(self) -> list[str]:
return [c.name for c in self.components.values() if c.status == "changed"]
@property
def unchanged_components(self) -> list[str]:
return [c.name for c in self.components.values() if c.status == "unchanged"]
@dataclass
class ManifestDiff:
"""Result of comparing two TheRock manifests."""
start_commit: str
end_commit: str
submodules: dict[str, Submodule] = field(default_factory=dict)
superrepos: dict[str, Superrepo] = field(default_factory=dict)
@property
def added(self) -> list[str]:
return [s.name for s in self.submodules.values() if s.status == "added"]
@property
def removed(self) -> list[str]:
return [s.name for s in self.submodules.values() if s.status == "removed"]
@property
def changed(self) -> list[str]:
return [s.name for s in self.submodules.values() if s.status == "changed"]
@property
def unchanged(self) -> list[str]:
return [s.name for s in self.submodules.values() if s.status == "unchanged"]
@property
def reverted(self) -> list[str]:
return [s.name for s in self.submodules.values() if s.status == "reverted"]
@property
def all_items(self) -> dict[str, Submodule | Superrepo]:
"""All submodules and superrepos combined."""
return {**self.submodules, **self.superrepos}
def get_status_groups(self) -> dict[str, list[str]]:
"""Return submodules grouped by status."""
groups: dict[str, list[str]] = {
"removed": [],
"added": [],
"unchanged": [],
"changed": [],
"reverted": [],
}
for item in self.all_items.values():
groups[item.status].append(item.name)
return groups
# =============================================================================
# CLI and Commit Resolution
# =============================================================================
def parse_args(argv: list[str] | None) -> argparse.Namespace:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(description="Generate manifest diff report")
parser.add_argument(
"--start", required=False, help="Start workflow ID or commit SHA"
)
parser.add_argument("--end", required=True, help="End workflow ID or commit SHA")
parser.add_argument(
"--find-last-run",
help=(
"Workflow filename (e.g. 'multi_arch_ci.yml'). When set, --start "
"is resolved as the head SHA of that workflow's most recent run "
"on --branch that concluded with 'success' or 'failure' "
"(cancelled / skipped / in-progress runs are ignored)."
),
)
parser.add_argument(
"--pr-base-ref",
help=(
"PR base branch name. When set, --start is resolved as the "
"merge-base between --end and this branch via the GitHub Compare "
"API. Rebase-safe."
),
)
parser.add_argument(
"--workflow-mode",
action="store_true",
help="Treat --start and --end as workflow run IDs instead of commit SHAs",
)
parser.add_argument(
"--branch",
default="main",
help=(
"Branch to scope --find-last-run lookups against (default: main). "
"Only consulted when --find-last-run is set; ignored otherwise."
),
)
parser.add_argument(
"--output-dir",
type=Path,
help="Output directory for the report (default: TheRock root directory)",
)
return parser.parse_args(argv)
def _optional_str(val: str | None) -> str | None:
"""Normalize optional string: None or blank -> None, otherwise stripped.
Workflow callers (e.g. GitHub Actions) may pass empty string for unused
optional inputs; treat those as not provided.
"""
if val is None:
return None
s = val.strip()
return s if s else None
def resolve_commits(
args: argparse.Namespace,
) -> tuple[str | None, str | None]:
"""Resolve start and end commit SHAs from arguments.
Returns ``(None, None)`` when --find-last-run finds no prior matching
run (graceful empty); caller should exit 0. Other API errors (e.g.
Compare API 404 on --pr-base-ref) propagate as GitHubAPIError; the
workflow's continue-on-error gate handles those.
"""
start = _optional_str(args.start)
find_last = _optional_str(args.find_last_run)
pr_base = _optional_str(args.pr_base_ref)
end = _optional_str(args.end)
if end is None:
raise ValueError("--end is required")
if start is None and find_last is None and pr_base is None:
raise ValueError(
"--start is required unless --find-last-run or --pr-base-ref is provided"
)
therock = f"{ROCM_ORG}/{THEROCK_REPO}"
# Resolve end first; --pr-base-ref needs end_sha to compute the merge-base.
if args.workflow_mode:
end_sha = gha_query_workflow_run_by_id(therock, end).get("head_sha")
else:
end_sha = end
# Resolve start, in priority order: --pr-base-ref, --find-last-run,
# --workflow-mode, then plain --start.
if pr_base is not None:
compare_url = (
f"https://api.github.com/repos/{therock}/compare/{pr_base}...{end_sha}"
)
compare = gha_send_request(compare_url)
start_sha = compare.get("merge_base_commit", {}).get("sha")
elif find_last is not None:
# Hardcoded to terminal-status: any run that ran to completion (success
# or failure) is acceptable — devs comparing to "the last run that
# actually ran" don't care whether it was green or red, only that it
# wasn't cancelled / skipped / in-progress.
accepted = {"success", "failure"}
last_run = gha_query_last_workflow_run(
therock,
find_last,
branch=args.branch,
accepted_statuses=accepted,
)
if last_run is None:
print(
f" [empty] No prior run of {find_last} on {args.branch} with "
f"conclusion in {sorted(accepted)} (likely a first-ever run)."
)
return None, None
start_sha = last_run["head_sha"]
elif args.workflow_mode:
start_sha = gha_query_workflow_run_by_id(therock, start).get("head_sha")
else:
start_sha = start
return start_sha, end_sha
# =============================================================================
# GitHub API Utilities
# =============================================================================
def get_api_base_from_url(url: str, fallback_name: str) -> str:
"""Convert git remote URL to GitHub API base URL."""
# To handle cases where urls are in ssh format, we need to convert them to https format
normalized = url.replace("git@github.com:", "https://github.com/")
parsed = urllib.parse.urlparse(normalized)
if parsed.netloc == "github.com":
path = parsed.path.lstrip("/")
if path.endswith(".git"):
path = path[:-4]
return f"https://api.github.com/repos/{path}"
print(f" Warning: Could not parse URL '{url}', using ROCm/{fallback_name}")
return f"{GITHUB_API_BASE}/{ROCM_ORG}/{fallback_name}"
def is_revert(old_sha: str, new_sha: str, api_base: str) -> bool:
"""Check if updating from old_sha to new_sha is a revert (going backwards).
Uses GitHub compare API: compare/{new_sha}...{old_sha}
- If old_sha is "ahead" of new_sha → old_sha came later → revert → return True
- If old_sha is "behind" new_sha → normal forward progress → return False
- If "diverged" → different branches, not a revert → return False
- If 404 → commits deleted from repo → return False (can't determine)
"""
try:
# If old_sha is "ahead" of new_sha, we're moving backwards (revert)
compare = gha_send_request(f"{api_base}/compare/{new_sha}...{old_sha}")
status = compare.get("status", "")
return status == "ahead"
except HTTPError as e:
# 404 = commits deleted from repo (hard orphaned, rare)
# Other errors = network issues, rate limits, etc.
if e.code == 404:
print(" (Commits deleted from repo - cannot determine revert status)")
return False
except (KeyError, ValueError, TypeError) as e:
print(f" Warning: Could not compare commits for revert detection: {e}")
return False
def fetch_commits_in_range(
repo_name: str,
start_sha: str,
end_sha: str,
api_base: str,
) -> list[dict]:
"""Fetch commits between two SHAs."""
commits: list[dict] = []
found_start = False
page = 1
print(f" Fetching commits for {repo_name}: {start_sha[:7]} -> {end_sha[:7]}")
while not found_start and page <= MAX_PAGES:
params = {"sha": end_sha, "per_page": PER_PAGE, "page": page}
url = f"{api_base}/commits?{urllib.parse.urlencode(params)}"
try:
data = gha_send_request(url)
if not data:
break
for commit in data:
if commit["sha"] == start_sha:
found_start = True
break
commits.append(commit)
if len(data) < PER_PAGE:
break
page += 1
except (KeyError, ValueError, TypeError) as e:
print(f" Error fetching commits: {e}")
break
# Fallback for diverged/off-branch commits (soft orphaned)
# This happens when commits exist in the repo but aren't in linear history,
# e.g., after a rebase where the branch moved but old commits weren't deleted.
# The compare API can still find commits unique to end_sha in these cases.
if not found_start:
try:
compare_url = f"{api_base}/compare/{start_sha}...{end_sha}"
compare_data = gha_send_request(compare_url)
if compare_data.get("status") == "diverged":
print(" (Commits are off-branch/diverged - using compare API)")
diverged_commits = compare_data.get("commits", [])
print(f" Found {len(diverged_commits)} commits unique to end")
return diverged_commits
except (KeyError, ValueError, TypeError, RuntimeError) as e:
print(f" Warning: Could not use compare API fallback: {e}")
print(f" Found {len(commits)} commits")
return commits
def fetch_superrepo_components(
repo_name: str, commit_sha: str, api_base: str
) -> list[str]:
"""Get component paths from superrepo at given commit.
Note: Some directories (e.g., shared/) may not exist at older commits.
404 errors are handled gracefully - directory is skipped.
"""
components: list[str] = []
for directory in SUPERREPO_COMPONENT_DIRS:
url = f"{api_base}/contents/{directory}?ref={commit_sha}"
try:
data = gha_send_request(url)
for item in data:
if item["type"] == "dir":
components.append(f"{directory}/{item['name']}")
except HTTPError as e:
if e.code == 404:
print(f" {directory}/ not found at this commit (skipping)")
else:
print(f" HTTP {e.code} fetching {directory}/")
except (KeyError, ValueError, TypeError) as e:
print(f" Failed to fetch {directory} folder: {e}")
return components
def fetch_commits_by_directory(
repo_name: str,
start_sha: str,
end_sha: str,
api_base: str,
directories: list[str],
) -> tuple[dict[str, list[dict]], list[dict]]:
"""Get commits allocated by directory."""
print(f" Getting commits by directories for {repo_name}")
all_commits = fetch_commits_in_range(
repo_name=repo_name,
start_sha=start_sha,
end_sha=end_sha,
api_base=api_base,
)
commit_shas_in_range = {c["sha"] for c in all_commits}
allocation: dict[str, list[dict]] = {}
seen_shas: set[str] = set()
for directory in directories:
dir_key = directory.rstrip("/")
allocation[dir_key] = []
page = 1
while page <= MAX_PAGES:
params = {
"sha": end_sha,
"path": directory,
"per_page": PER_PAGE,
"page": page,
}
url = f"{api_base}/commits?{urllib.parse.urlencode(params)}"
try:
data = gha_send_request(url)
if not data:
break
commits_found = 0
for commit in data:
sha = commit["sha"]
if sha in commit_shas_in_range:
allocation[dir_key].append(commit)
seen_shas.add(sha)
commits_found += 1
if sha == start_sha:
break
if commits_found == 0 and page > 1:
break
if len(data) < PER_PAGE:
break
page += 1
except (KeyError, ValueError, TypeError) as e:
print(f" Error querying directory {directory}: {e}")
break
if allocation[dir_key]:
print(f" {dir_key}: {len(allocation[dir_key])} commits")
# Unassigned commits
unassigned = [c for c in all_commits if c["sha"] not in seen_shas]
if unassigned:
allocation[UNASSIGNED_KEY] = unassigned
print(f" {UNASSIGNED_KEY}: {len(unassigned)} commits")
return allocation, all_commits
# =============================================================================
# Manifest Loading
# =============================================================================
def load_submodules_at_commit(commit_sha: str) -> dict[str, dict[str, str]]:
"""Load all submodules from TheRock at a specific commit.
Uses build_manifest_schema from generate_therock_manifest to get submodule
information and converts it to the expected format.
Note: Requires the repository to be checked out with full history
(fetch-depth: 0) for historical commits to be accessible via local git.
"""
try:
manifest = build_manifest_schema(THEROCK_DIR, commit_sha)
submodules: dict[str, dict[str, str]] = {}
for entry in manifest.get("submodules", []):
name = entry["submodule_name"]
url = entry["submodule_url"]
pin_sha = entry.get("pin_sha")
if pin_sha:
submodules[name] = {
"sha": pin_sha,
"api_base": get_api_base_from_url(url, name),
"branch": None, # Branch info not in manifest schema
}
print(f"Found {len(submodules)} submodules in manifest for {commit_sha[:8]}")
return submodules
except (RuntimeError, KeyError) as e:
print(f"Error loading submodules for commit {commit_sha}: {e}")
return {}
# =============================================================================
# Status Determination
# =============================================================================
def determine_status(
old_sha: str | None,
new_sha: str | None,
api_base: str,
) -> tuple[str, str, str]:
"""Determine submodule status. Returns (status, fetch_start, fetch_end)."""
if old_sha and not new_sha:
return "removed", "", ""
if new_sha and not old_sha:
return "added", "", new_sha
if old_sha == new_sha:
return "unchanged", "", ""
# Check for revert (is_revert handles orphaned/404 gracefully by returning False)
if is_revert(old_sha, new_sha, api_base):
return "reverted", new_sha, old_sha
return "changed", old_sha, new_sha
# =============================================================================
# Submodule Processing
# =============================================================================
def process_regular_submodule(
name: str,
start_data: dict[str, str] | None,
end_data: dict[str, str] | None,
) -> Submodule:
"""Process a regular (non-superrepo) submodule."""
old_sha = start_data["sha"] if start_data else None
new_sha = end_data["sha"] if end_data else None
data = end_data or start_data or {}
api_base = data.get("api_base", "")
branch = data.get("branch", "main")
status, fetch_start, fetch_end = determine_status(old_sha, new_sha, api_base)
submodule = Submodule(
name=name,
sha=new_sha or old_sha or "",
api_base=api_base,
branch=branch,
status=status,
start_sha=old_sha or "",
end_sha=new_sha or "",
)
if status == "removed":
print(f" {name}: REMOVED")
elif status == "added":
print(f" {name}: ADDED -> {fetch_end[:7]}")
# For a newly added submodule, we need to fetch only the tip commit
try:
tip_commit = gha_send_request(f"{api_base}/commits/{fetch_end}")
submodule.commits = [tip_commit] if tip_commit else []
except (KeyError, ValueError, TypeError) as e:
print(f" Warning: Could not fetch tip commit: {e}")
elif status == "unchanged":
print(f" {name}: UNCHANGED")
elif status in ("changed", "reverted"):
label = "REVERTED" if status == "reverted" else "CHANGED"
print(f" {name}: {label} {fetch_start[:7]} -> {fetch_end[:7]}")
submodule.commits = fetch_commits_in_range(
repo_name=name,
start_sha=fetch_start,
end_sha=fetch_end,
api_base=api_base,
)
return submodule
class SuperrepoProcessor:
"""Processes a superrepo with component-level analysis."""
def __init__(
self,
name: str,
start_data: dict[str, str] | None,
end_data: dict[str, str] | None,
):
self.name = name
self.start_data = start_data
self.end_data = end_data
self.old_sha = start_data["sha"] if start_data else None
self.new_sha = end_data["sha"] if end_data else None
data = end_data or start_data or {}
self.api_base = data.get("api_base", "")
self.branch = data.get("branch", "main")
self.superrepo: Superrepo | None = None
self.fetch_start: str | None = None
self.fetch_end: str | None = None
def process(self) -> Superrepo:
"""Process the superrepo and return the result."""
self.init_superrepo()
print(f" {self.name}: SUPERREPO ({self.superrepo.status})")
if self.superrepo.status == "removed":
return self.handle_removed()
elif self.superrepo.status == "added":
return self.handle_added()
elif self.superrepo.status == "unchanged":
return self.handle_unchanged()
else:
return self.handle_changed_or_reverted()
def init_superrepo(self) -> None:
"""Initialize the Superrepo object with basic info."""
status, self.fetch_start, self.fetch_end = determine_status(
self.old_sha, self.new_sha, self.api_base
)
self.superrepo = Superrepo(
name=self.name,
sha=self.new_sha or self.old_sha or "",
api_base=self.api_base,
branch=self.branch,
status=status,
start_sha=self.old_sha or "",
end_sha=self.new_sha or "",
)
def handle_removed(self) -> Superrepo:
"""Handle a removed superrepo."""
start_components = fetch_superrepo_components(
self.name, self.old_sha, self.api_base
)
print(f" Removed superrepo had {len(start_components)} components")
for comp_path in start_components:
self.add_component(comp_path, status="removed")
return self.superrepo
def handle_added(self) -> Superrepo:
"""Handle a newly added superrepo."""
end_components = fetch_superrepo_components(
self.name, self.new_sha, self.api_base
)
print(f" Added superrepo with {len(end_components)} components")
print(f" Fetching tip commits for each component...")
for comp_path in end_components:
tip_commits = self.fetch_tip_commit(comp_path)
self.add_component(comp_path, status="added", commits=tip_commits)
return self.superrepo
def handle_unchanged(self) -> Superrepo:
"""Handle an unchanged superrepo."""
end_components = fetch_superrepo_components(
self.name, self.new_sha, self.api_base
)
print(f" Unchanged superrepo with {len(end_components)} components")
for comp_path in end_components:
self.add_component(comp_path, status="unchanged")
return self.superrepo
def handle_changed_or_reverted(self) -> Superrepo:
"""Handle a changed or reverted superrepo with full commit analysis."""
start_components = (
fetch_superrepo_components(self.name, self.old_sha, self.api_base)
if self.old_sha
else []
)
end_components = (
fetch_superrepo_components(self.name, self.new_sha, self.api_base)
if self.new_sha
else []
)
start_set = set(start_components)
end_set = set(end_components)
added_paths = end_set - start_set
removed_paths = start_set - end_set
print(
f" Components: {len(start_components)} -> {len(end_components)} "
f"(+{len(added_paths)} -{len(removed_paths)})"
)
all_components = start_set | end_set
directories = [c + "/" if not c.endswith("/") else c for c in all_components]
allocation, all_commits = fetch_commits_by_directory(
repo_name=self.name,
start_sha=self.fetch_start,
end_sha=self.fetch_end,
api_base=self.api_base,
directories=directories,
)
self.superrepo.all_commits = all_commits
self.superrepo.commit_allocation = allocation
for comp_path in all_components:
comp_status, comp_commits = self.determine_component_status(
comp_path, added_paths, removed_paths, allocation
)
self.add_component(comp_path, status=comp_status, commits=comp_commits)
return self.superrepo
def determine_component_status(
self,
comp_path: str,
added_paths: set[str],
removed_paths: set[str],
allocation: dict[str, list[dict]],
) -> tuple[str, list[dict]]:
"""Determine the status and commits for a component."""
comp_key = comp_path.rstrip("/")
if comp_path in added_paths:
comp_commits = allocation.get(comp_key, [])
if comp_commits:
comp_commits = [comp_commits[0]]
return "added", comp_commits
elif comp_path in removed_paths:
return "removed", []
elif comp_key in allocation and allocation[comp_key]:
return "changed", allocation.get(comp_key, [])
else:
return "unchanged", []
def fetch_tip_commit(self, comp_path: str) -> list[dict]:
"""Fetch the tip commit for a component."""
comp_name = comp_path.split("/")[-1]
try:
params = {"sha": self.new_sha, "path": comp_path, "per_page": 1}
url = f"{self.api_base}/commits?{urllib.parse.urlencode(params)}"
tip_data = gha_send_request(url)
if tip_data:
return [tip_data[0]]
except (KeyError, ValueError, TypeError, HTTPError) as e:
print(f" Warning: Could not fetch tip for {comp_name}: {e}")
return []
def add_component(
self, comp_path: str, status: str, commits: list[dict] | None = None
) -> None:
"""Add a component to the superrepo."""
comp_name = comp_path.split("/")[-1]
self.superrepo.components[comp_path] = Component(
path=comp_path, name=comp_name, status=status, commits=commits or []
)
def process_superrepo(
name: str,
start_data: dict[str, str] | None,
end_data: dict[str, str] | None,
) -> Superrepo:
"""Process a superrepo with component-level analysis."""
return SuperrepoProcessor(name, start_data, end_data).process()
# =============================================================================
# ManifestDiff Construction
# =============================================================================
def compare_manifests(start_commit: str, end_commit: str) -> ManifestDiff:
"""Compare two TheRock commits and return a ManifestDiff."""
print(f"\nComparing commits: {start_commit[:7]} -> {end_commit[:7]}")
print("\n=== Getting submodules for START commit ===")
start_subs = load_submodules_at_commit(start_commit)
print(f"\nFound {len(start_subs)} submodules for START commit")
print("\n=== Getting submodules for END commit ===")
end_subs = load_submodules_at_commit(end_commit)
print(f"\nFound {len(end_subs)} submodules for END commit")
diff = ManifestDiff(start_commit=start_commit, end_commit=end_commit)
all_names = set(start_subs.keys()) | set(end_subs.keys())
print("\n=== Processing Submodules ===")
for name in sorted(all_names):
start_data = start_subs.get(name)
end_data = end_subs.get(name)
if name in SUPERREPO_NAMES:
diff.superrepos[name] = process_superrepo(name, start_data, end_data)
else:
diff.submodules[name] = process_regular_submodule(
name, start_data, end_data
)
return diff
# =============================================================================
# HTML Helpers
# =============================================================================
def format_commit_date(date_string: str) -> str:
"""Format ISO date string to readable format."""
if date_string == "Unknown" or not date_string:
return "Unknown"
try:
dt = datetime.fromisoformat(date_string.replace("Z", "+00:00"))
return dt.strftime("%b %d, %Y")
except (ValueError, TypeError):
return date_string
def create_commit_badge(sha: str, repo_name: str) -> str:
"""Create HTML badge for a commit SHA."""
short_sha = sha[:7] if sha not in ("-", "N/A", "") else sha or "N/A"
url = f"https://github.com/ROCm/{repo_name}/commit/{sha}"
return (
f""
f"{short_sha}"
)
def extract_commit_data(commit: dict) -> dict[str, str]:
"""Extract commit data from GitHub API response."""
return {
"sha": commit.get("sha", "-"),
"message": commit.get("commit", {}).get("message", "-").split("\n")[0],
"author": commit.get("commit", {}).get("author", {}).get("name", "Unknown"),
"date": format_commit_date(
commit.get("commit", {}).get("author", {}).get("date", "Unknown")
),
}
def create_commit_item_html(commit: dict, repo_name: str) -> str:
"""Create HTML for a single commit item."""
data = extract_commit_data(commit)
badge = create_commit_badge(data["sha"], repo_name)
return (
f""
f"
{badge} {data['message']}
"
f"
{data['date']} | {data['author']}
"
f"
Component removed in this version
'
elif status == "reverted":
# Build informative revert message with commit links
if start_sha and end_sha:
start_badge = create_commit_badge(start_sha[:7], repo_name)
end_badge = create_commit_badge(end_sha[:7], repo_name)
revert_msg = f"Reverted from {start_badge} back to {end_badge}."
else:
# For components within a reverted superrepo (no individual SHAs)
revert_msg = "Commits being undone by this revert."
content = (
f'" + "".join(commit_items)
)
elif not commit_items:
content = 'Component has no commits in this range (Superrepo Component Unchanged)
'
else:
content = "".join(commit_items)
# Add background classes for status styling
classes = ["commit-list"]
if status == "newly_added":
classes.append("newly-added-bg")
elif status == "removed":
classes.append("removed-bg")
elif status == "reverted":
classes.append("reverted-bg")
return f"{content}
"
def create_table(headers: list[str], rows: list[str]) -> str:
"""Create HTML table."""
header_html = "".join(f"{h} | " for h in headers)
return (
""
)
# =============================================================================
# Summary HTML Generation
# =============================================================================
def generate_summary_html(items: dict[str, list[str]], summary_type: str) -> str:
"""Generate HTML summary lists."""
total = sum(len(v) for v in items.values())
if total == 0:
return ""
categories = [
("added", "Newly Added"),
("removed", "Removed"),
("changed", "Changed"),
("unchanged", "Unchanged"),
("reverted", "Reverted"),
]
parts = []
for cat, label in categories:
cat_items = items.get(cat, [])
if not cat_items:
continue
list_html = "".join(f"{i}" for i in sorted(cat_items))
parts.append(
f''
f"
{label} {summary_type.title()} ({len(cat_items)}/{total}):
"
f'
{self.superrepo.name}: REMOVED
"
return f"{self.superrepo.name}: No components
"
def build_status_banner(self) -> str:
"""Build the status banner for added/removed/reverted superrepos."""
if self.superrepo.status == "added":
replaced_note = ""
if self.removed_submodules:
replaced_list = ", ".join(sorted(self.removed_submodules))
replaced_note = (
f"Replaces direct submodules: "
f"{replaced_list}
"
)
return (
""
f"NEWLY ADDED SUPERREPO - "
f"This superrepo was newly added with {len(self.superrepo.components)} components. "
f"Showing tip commit for each component.{replaced_note}
"
)
elif self.superrepo.status == "removed":
return (
""
f"REMOVED SUPERREPO - "
f"This superrepo was removed. It previously contained "
f"{len(self.superrepo.components)} components.
"
)
elif self.superrepo.status == "reverted":
start_badge = create_commit_badge(
self.superrepo.start_sha[:7], self.superrepo.name
)
end_badge = create_commit_badge(
self.superrepo.end_sha[:7], self.superrepo.name
)
return (
""
f"REVERTED SUPERREPO - "
f"This superrepo was reverted from {start_badge} back to {end_badge}. "
f"Commits shown below are being undone by this revert.
"
)
return ""
def build_commit_mapping(self) -> None:
"""Build mapping from commit SHA to component names."""
for comp in self.superrepo.components.values():
for commit in comp.commits:
sha = commit.get("sha", "-")
if sha not in self.commit_to_projects:
self.commit_to_projects[sha] = set()
self.commit_to_projects[sha].add(comp.name)
def build_component_table(self) -> str:
"""Build the component table HTML."""
rows = []
for comp in sorted(self.superrepo.components.values(), key=lambda c: c.path):
row_classes = []
status_class = None
if comp.status == "added":
status_class = "newly_added"
elif comp.status == "removed":
status_class = "removed"
elif comp.status == "unchanged":
row_classes.append("unchanged-row")
elif self.superrepo.status == "reverted" and comp.commits:
status_class = "reverted"
row_classes.append("component-row")
commit_html = create_commit_list_html(
comp.commits, self.superrepo.name, status_class
)
row_class_attr = f" class='{' '.join(row_classes)}'" if row_classes else ""
data_component = html.escape(comp.name, quote=True)
rows.append(
f"{comp.name} | {commit_html} | "
)
return create_table(["Component", "Commits"], rows)
def build_history_table(self) -> str:
"""Build the commit history table HTML."""
if not self.superrepo.all_commits:
return ""
history_rows = []
for commit in self.superrepo.all_commits:
data = extract_commit_data(commit)
projects = (
", ".join(sorted(self.commit_to_projects.get(data["sha"], [])))
or "Unassigned"
)
badge = create_commit_badge(data["sha"], self.superrepo.name)
history_rows.append(
f""
f"| {data['date']} | "
f"{badge} | "
f"{data['author']} | "
f"{projects} | "
f"{data['message']} | "
f"
"
)
if not history_rows:
return ""
return (
""
"Commit History (newest to oldest):
"
""
"| Date | SHA | "
"Author | Project(s) | "
"Message |
|---|
" + "".join(history_rows) + "
"
)
def generate_superrepo_html(
superrepo: Superrepo, removed_submodules: list[str] | None = None
) -> str:
"""Generate HTML for a superrepo section."""
return SuperrepoHtmlBuilder(superrepo, removed_submodules).build()
# =============================================================================
# Non-Superrepo HTML Generation
# =============================================================================
def generate_non_superrepo_html(diff: ManifestDiff) -> str:
"""Generate HTML for non-superrepo submodules table."""
submodules = list(diff.submodules.values())
print(f" Generating HTML for {len(submodules)} non-superrepo submodules")
if not submodules:
return "No non-superrepo submodules found
"
# Banner for removed submodules (moved to superrepo)
removed_banner = ""
removed_subs = [s for s in submodules if s.status == "removed"]
if removed_subs:
# Check if any superrepo was added (likely the destination)
added_superrepos = [
s.name for s in diff.superrepos.values() if s.status == "added"
]
destination = (
f" (moved to {', '.join(added_superrepos)})" if added_superrepos else ""
)
removed_names = ", ".join(sorted(s.name for s in removed_subs))
removed_banner = (
""
f"REMOVED SUBMODULES ({len(removed_subs)}){destination}
"
f"{removed_names}
"
)
rows = []
for sub in sorted(submodules, key=lambda s: s.name):
# Show range of commits actually displayed (oldest → newest)
if sub.commits:
oldest_sha = sub.commits[-1].get("sha", "-")
newest_sha = sub.commits[0].get("sha", "-")
start_badge = create_commit_badge(oldest_sha, sub.name)
end_badge = create_commit_badge(newest_sha, sub.name)
else:
start_badge = create_commit_badge(sub.start_sha or "-", sub.name)
end_badge = create_commit_badge(sub.end_sha or "-", sub.name)
range_html = f"Range: {start_badge} -> {end_badge}
"
row_classes = []
status_class = None
if sub.status == "added":
status_class = "newly_added"
elif sub.status == "removed":
status_class = "removed"
elif sub.status == "reverted":
status_class = "reverted"
elif sub.status == "unchanged":
row_classes.append("unchanged-row")
# Pass start/end SHAs for reverted status message
commit_html = create_commit_list_html(
sub.commits,
sub.name,
status_class,
start_sha=sub.start_sha if sub.status == "reverted" else None,
end_sha=sub.end_sha if sub.status == "reverted" else None,
)
full_content = range_html + commit_html
row_classes.append("component-row")
row_class_attr = f" class='{' '.join(row_classes)}'" if row_classes else ""
data_component = html.escape(sub.name, quote=True)
rows.append(
f"{sub.name} | {full_content} | "
)
return removed_banner + create_table(["Submodule", "Commits"], rows)
# =============================================================================
# Report Generation
# =============================================================================
class HtmlReportGenerator:
"""Generates HTML reports from ManifestDiff data."""
def __init__(self, diff: ManifestDiff, output_dir: Path | None = None):
"""Initialize the generator.
Args:
diff: The manifest diff to generate the report from.
output_dir: Optional output directory. If provided, creates the directory
and writes the report there. Otherwise writes to TheRock root.
"""
self.diff = diff
self.output_dir = output_dir
self.html = ""
self.report_path: Path | None = None
def generate(self) -> Path:
"""Generate the HTML report.
Returns:
Path to the generated report file.
"""
print("\n=== Writing HTML Report ===")
self.load_template()
self.inject_commit_range()
self.inject_summaries()
self.inject_section_badges()
self.inject_section_content()
return self.write_report()
def load_template(self) -> None:
"""Load the HTML template and determine output path."""
if self.output_dir:
self.output_dir.mkdir(parents=True, exist_ok=True)
self.report_path = self.output_dir / "TheRockReport.html"
else:
self.report_path = HTML_REPORT_PATH
if not HTML_TEMPLATE_PATH.exists():
print(f" ERROR: Template not found at {HTML_TEMPLATE_PATH}")
raise FileNotFoundError(f"Template not found at {HTML_TEMPLATE_PATH}")
self.html = HTML_TEMPLATE_PATH.read_text(encoding="utf-8")
def inject_commit_range(self) -> None:
"""Inject TheRock start and end commit badges into the template."""
self.html = self.html.replace(
'START_COMMIT',
f'{self.diff.start_commit[:8]}',
)
self.html = self.html.replace(
'END_COMMIT',
f'{self.diff.end_commit[:8]}',
)
def inject_summaries(self) -> None:
"""Inject submodule and superrepo summary sections into the template."""
# Submodule summary
status_groups = self.diff.get_status_groups()
submodule_summary = generate_summary_html(status_groups, "submodules")
self.html = self.html.replace(
'',
f'{submodule_summary}
',
)
# Superrepo summaries
superrepo_parts = []
for superrepo in self.diff.superrepos.values():
comp_groups = {
"added": superrepo.added_components,
"removed": superrepo.removed_components,
"changed": superrepo.changed_components,
"unchanged": superrepo.unchanged_components,
}
comp_summary = generate_summary_html(comp_groups, "components")
if comp_summary:
superrepo_parts.append(
f"{superrepo.name.title()} Components
"
f"{comp_summary}"
)
self.html = self.html.replace(
'',
f'{"".join(superrepo_parts)}
',
)
def inject_section_badges(self) -> None:
"""Inject commit badges for each section header in the template."""
# Superrepo section badges
for repo_name in ["rocm-libraries", "rocm-systems"]:
superrepo = self.diff.superrepos.get(repo_name)
start_sha = superrepo.start_sha if superrepo else ""
end_sha = superrepo.end_sha if superrepo else ""
start_badge = (
create_commit_badge(start_sha, repo_name) if start_sha else "N/A"
)
end_badge = create_commit_badge(end_sha, repo_name) if end_sha else "N/A"
self.html = self.html.replace(
f'',
f'{start_badge}',
)
self.html = self.html.replace(
f'',
f'{end_badge}',
)
# Non-superrepo section badges
start_badge = create_commit_badge(self.diff.start_commit, THEROCK_REPO)
end_badge = create_commit_badge(self.diff.end_commit, THEROCK_REPO)
self.html = self.html.replace(
'',
f'{start_badge}',
)
self.html = self.html.replace(
'',
f'{end_badge}',
)
def inject_section_content(self) -> None:
"""Inject detailed commit content for each section in the template."""
# Superrepo sections
for repo_name in ["rocm-libraries", "rocm-systems"]:
superrepo = self.diff.superrepos.get(repo_name)
if superrepo:
# Pass removed submodules if this superrepo was newly added
removed_subs = (
self.diff.removed if superrepo.status == "added" else None
)
content = generate_superrepo_html(
superrepo, removed_submodules=removed_subs
)
else:
content = ""
self.html = self.html.replace(
f'\n
',
f'\n {content}\n
',
)
# Non-superrepo section
non_superrepo_html = generate_non_superrepo_html(self.diff)
self.html = self.html.replace(
'\n
',
f'\n {non_superrepo_html}\n
',
)
def write_report(self) -> Path:
"""Write the HTML report to disk and return the path."""
self.report_path.write_text(self.html, encoding="utf-8")
if not self.report_path.exists() or self.report_path.stat().st_size == 0:
raise RuntimeError(f"Failed to write HTML report to {self.report_path}")
print(f" Report written to: {self.report_path}")
return self.report_path
def generate_html_report(diff: ManifestDiff, output_dir: Path | None = None) -> Path:
"""Generate TheRockReport.html from template.
Args:
diff: The manifest diff to generate the report from.
output_dir: Optional output directory. If provided, creates the directory
and writes the report there. Otherwise writes to TheRock root.
Returns:
Path to the generated report file.
"""
return HtmlReportGenerator(diff, output_dir).generate()
def generate_step_summary(diff: ManifestDiff) -> None:
"""Generate GitHub Actions step summary using the shared utility."""
summary = "# TheRock Manifest Diff Report\n\n"
summary += (
f"**Commit Range:** `{diff.start_commit[:8]}` -> `{diff.end_commit[:8]}`\n\n"
)
summary += "## Submodule Changes\n\n"
# Submodule changes
status_configs = [
("added", "Added"),
("removed", "Removed"),
("changed", "Changed"),
("reverted", "Reverted"),
]
status_groups = diff.get_status_groups()
for status, title in status_configs:
items = status_groups.get(status, [])
if items:
summary += f"### {title} ({len(items)})\n"
for name in sorted(items):
sub = diff.all_items.get(name)
if status == "changed" and sub:
summary += f"- `{name}` ({len(sub.commits)} commits)\n"
else:
summary += f"- `{name}`\n"
summary += "\n"
unchanged = status_groups.get("unchanged", [])
if unchanged:
summary += f"Unchanged ({len(unchanged)})
\n\n"
for n in sorted(unchanged):
summary += f"- `{n}`\n"
summary += "\n