TheRock/build_tools/_therock_utils/workflow_outputs.py

"""CI workflow outputs layout specification.

This module defines the canonical directory structure for all outputs from a
GitHub Actions workflow run. All tools that read or write workflow outputs
should use this module to compute paths.

See docs/development/workflow_outputs.md for the full layout reference.

Note: this file is NOT bundled into the Lambda deployment package (only
storage_backend.py and storage_location.py are). Top-level imports here
do not need to be Lambda-safe, but keep optional dependencies deferred
(imported inside the function that needs them) so this module remains
importable in environments where those packages are not installed.

A "workflow output" is anything produced by a CI workflow run:
- Build artifacts (.tar.xz, .tar.zst archives)
- Logs (.log files, ninja_logs.tar.gz)
- Manifests (therock_manifest.json)
- Python packages (.whl, .tar.gz)
- Reports (build_observability.html, test reports)

Usage::

    from _therock_utils.workflow_outputs import WorkflowOutputRoot

    # Inside a CI workflow (env vars provide bucket info, no API call)
    root = WorkflowOutputRoot.from_workflow_run(run_id="12345", platform="linux")

    # Fetching artifacts from another run (API call for fork/cutover detection)
    root = WorkflowOutputRoot.from_workflow_run(
        run_id="12345", platform="linux", lookup_workflow_run=True
    )

    # For local development/testing
    root = WorkflowOutputRoot.for_local(run_id="local", platform="linux")

    # Get locations for various outputs
    loc = root.artifact("blas_lib_gfx94X.tar.xz")
    print(loc.s3_uri)       # s3://therock-ci-artifacts/12345-linux/blas_lib_gfx94X.tar.xz
    print(loc.https_url)    # https://therock-ci-artifacts.s3.amazonaws.com/...
    print(loc.local_path(Path("/tmp/staging")))  # /tmp/staging/12345-linux/...
"""

import os
import sys
from dataclasses import dataclass
from pathlib import Path
import platform as platform_module

# Add build_tools to path for sibling package imports.
sys.path.insert(0, os.fspath(Path(__file__).parent.parent))

from _therock_utils.storage_location import StorageLocation
from _therock_utils.s3_buckets import get_artifacts_bucket_config_for_workflow_run


# ---------------------------------------------------------------------------
# WorkflowOutputRoot
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class WorkflowOutputRoot:
    """Root location for all outputs from a single CI workflow run.

    This is the single source of truth for computing paths to workflow outputs.
    Each method returns a `StorageLocation` that can be resolved to
    S3 URIs, HTTPS URLs, or local paths as needed.

    The class is immutable (frozen) to ensure path computation is deterministic.
    """

    bucket: str
    """S3 bucket name (e.g., 'therock-ci-artifacts')."""

    external_repo: str
    """Repository prefix (e.g., '' for ROCm/TheRock, 'owner-repo/' for forks)."""

    run_id: str
    """GitHub Actions workflow run ID (e.g., '12345678901')."""

    platform: str
    """Platform name ('linux' or 'windows')."""

    # -- Root -------------------------------------------------------------------

    @property
    def prefix(self) -> str:
        """Relative path prefix for this run (no trailing slash).

        This is the common root for all outputs from this run.
        """
        return f"{self.external_repo}{self.run_id}-{self.platform}"

    def root(self) -> StorageLocation:
        """Location for the run output root (where build artifacts live)."""
        return StorageLocation(self.bucket, self.prefix)

    # -- Build artifacts --------------------------------------------------------

    def artifact(self, filename: str) -> StorageLocation:
        """Location for a build artifact file.

        Args:
            filename: Artifact filename (e.g., 'blas_lib_gfx94X.tar.xz')
        """
        return StorageLocation(self.bucket, f"{self.prefix}/{filename}")

    def artifact_index(self) -> StorageLocation:
        """Location for the artifact index HTML (server-side generated)."""
        return StorageLocation(self.bucket, f"{self.prefix}/index.html")

    # -- Logs -------------------------------------------------------------------
    #
    # The log directory contains all build logs, reports, and profiling data
    # for an artifact group. log_dir() gives the directory root; the
    # remaining methods address well-known files within that subtree.

    def log_dir(self, artifact_group: str) -> StorageLocation:
        """Location for a log directory.

        The directory typically contains build.log, ninja_logs.tar.gz,
        build_observability.html (when generated), index.html, and a
        therock-build-prof/ subdirectory with resource profiling data.

        Args:
            artifact_group: Build variant (e.g., 'gfx94X-dcgpu')
        """
        return StorageLocation(self.bucket, f"{self.prefix}/logs/{artifact_group}")

    def log_stage_dir(
        self, stage_name: str, amdgpu_family: str = ""
    ) -> StorageLocation:
        """Location for a multi-arch stage log directory.

        Multi-arch CI uploads logs per stage (and optionally per GPU family)
        rather than per artifact_group. Generic stages get a single directory;
        per-arch stages (e.g., math-libs) get a subdirectory per family.

        Args:
            stage_name: Build stage (e.g., 'compiler-runtime', 'math-libs')
            amdgpu_family: GPU family (e.g., 'gfx1151'). Empty for generic stages.
        """
        if amdgpu_family:
            return StorageLocation(
                self.bucket, f"{self.prefix}/logs/{stage_name}/{amdgpu_family}"
            )
        return StorageLocation(self.bucket, f"{self.prefix}/logs/{stage_name}")

    def log_file(self, artifact_group: str, filename: str) -> StorageLocation:
        """Location for a specific file within the log_dir() subtree.

        Args:
            artifact_group: Build variant (e.g., 'gfx94X-dcgpu')
            filename: Log filename (e.g., 'build.log', 'ninja_logs.tar.gz')
        """
        return StorageLocation(
            self.bucket, f"{self.prefix}/logs/{artifact_group}/{filename}"
        )

    def log_index(self, artifact_group: str) -> StorageLocation:
        """Location for the log directory index HTML (within log_dir())."""
        return StorageLocation(
            self.bucket, f"{self.prefix}/logs/{artifact_group}/index.html"
        )

    def log_root_index(self) -> StorageLocation:
        """Location for the root log index HTML (server-side generated)."""
        return StorageLocation(self.bucket, f"{self.prefix}/logs/index.html")

    def build_observability(self, artifact_group: str) -> StorageLocation:
        """Location for build observability HTML (within log_dir())."""
        return StorageLocation(
            self.bucket,
            f"{self.prefix}/logs/{artifact_group}/build_observability.html",
        )

    # -- Manifests --------------------------------------------------------------

    def manifest_dir(self, artifact_group: str) -> StorageLocation:
        """Location for the manifests directory for an artifact group.

        Args:
            artifact_group: Build variant (e.g., 'gfx94X-dcgpu')
        """
        return StorageLocation(self.bucket, f"{self.prefix}/manifests/{artifact_group}")

    def manifest(self, artifact_group: str) -> StorageLocation:
        """Location for therock_manifest.json.

        Args:
            artifact_group: Build variant (e.g., 'gfx94X-dcgpu')
        """
        return StorageLocation(
            self.bucket,
            f"{self.prefix}/manifests/{artifact_group}/therock_manifest.json",
        )

    def manifest_root(self) -> StorageLocation:
        """Location for the workflow-level therock_manifest.json."""
        return StorageLocation(
            self.bucket,
            f"{self.prefix}/manifests/therock_manifest.json",
        )

    def manifests_index(self) -> StorageLocation:
        """Location for the workflow-level manifests index."""
        return StorageLocation(
            self.bucket,
            f"{self.prefix}/manifests/index.html",
        )

    # -- Native packages --------------------------------------------------------

    def native_linux_packages(self, pkg_type: str) -> StorageLocation:
        """Location for the native Linux package repository directory.

        Returns ``StorageLocation`` at ``{run_id}-linux/packages/{pkg_type}``
        (e.g. ``12345678901-linux/packages/deb``).

        The contents under this prefix follow standard repository layouts
        (not loose files): deb repos use APT layout (``pool/main/`` +
        ``dists/stable/``); rpm repos place packages under ``x86_64/`` with
        ``repodata/`` alongside. See ``upload_package_repo.py`` for the exact
        on-disk and S3 layout.

        Args:
            pkg_type: Package type ('deb' or 'rpm').
        """
        return StorageLocation(self.bucket, f"{self.prefix}/packages/{pkg_type}")

    # -- Python packages --------------------------------------------------------

    def python_packages(self, artifact_group: str = "") -> StorageLocation:
        """Location for the Python packages directory.

        Args:
            artifact_group: Build variant (e.g., 'gfx110X-all'). If empty,
                packages are stored directly under python/ (used for
                multi-arch builds where run_id already uniquely identifies
                the build).
        """
        suffix = f"/{artifact_group}" if artifact_group else ""
        return StorageLocation(self.bucket, f"{self.prefix}/python{suffix}")

    # -- Tarballs ---------------------------------------------------------------

    def tarballs(self) -> StorageLocation:
        """Location for the tarballs directory."""
        return StorageLocation(self.bucket, f"{self.prefix}/tarballs")

    def tarball(self, filename: str) -> StorageLocation:
        """Location for a specific tarball file."""
        return StorageLocation(self.bucket, f"{self.prefix}/tarballs/{filename}")

    # -- Factories --------------------------------------------------------------

    @classmethod
    def from_workflow_run(
        cls,
        run_id: str,
        platform: str,
        github_repository: str | None = None,
        workflow_run: dict | None = None,
        lookup_workflow_run: bool = False,
        release_type: str | None = None,
    ) -> "WorkflowOutputRoot":
        """Create from CI workflow context.

        Determines the S3 bucket and external_repo prefix from repository
        metadata and environment variables.

        Args:
            run_id: GitHub Actions workflow run ID.
            platform: Platform name ('linux' or 'windows').
            github_repository: Repository in 'owner/repo' format. If None,
                reads GITHUB_REPOSITORY env var (default: 'ROCm/TheRock').
            workflow_run: Optional workflow run dict from GitHub API. If
                provided, uses it directly for fork detection and bucket
                cutover dating (no API call).
            lookup_workflow_run: If True and ``workflow_run`` is not provided,
                fetches the workflow run from the GitHub API using ``run_id``.
                Most callers running inside their own CI workflow do not need
                this — environment variables suffice. Set this when looking up
                another repository's workflow run (e.g. fetching artifacts).
            release_type: Release type override (e.g. "ci", "dev", "nightly"). If
                None, falls back to the RELEASE_TYPE environment variable.
        """
        workflow_run_id = (
            run_id if lookup_workflow_run and workflow_run is None else None
        )
        external_repo, bucket = _retrieve_bucket_info(
            github_repository=github_repository,
            workflow_run_id=workflow_run_id,
            workflow_run=workflow_run,
            release_type=release_type,
        )
        return cls(
            bucket=bucket,
            external_repo=external_repo,
            run_id=run_id,
            platform=platform,
        )

    @classmethod
    def for_local(
        cls,
        run_id: str = "local",
        platform: str | None = None,
        bucket: str = "local",
    ) -> "WorkflowOutputRoot":
        """Create for local development/testing.

        Args:
            run_id: Run identifier (default: 'local').
            platform: Platform name. If None, detects from current system.
            bucket: Bucket name placeholder (default: 'local').
        """
        if platform is None:
            platform = platform_module.system().lower()
        return cls(
            bucket=bucket,
            external_repo="",
            run_id=run_id,
            platform=platform,
        )


def _retrieve_bucket_info(
    github_repository: str | None = None,
    workflow_run_id: str | None = None,
    workflow_run: dict | None = None,
    release_type: str | None = None,
) -> tuple[str, str]:
    """Determine S3 bucket and external_repo prefix for a workflow run.

    This is an internal implementation detail — use
    `WorkflowOutputRoot.from_workflow_run` instead.

    Returns:
        Tuple of ``(external_repo, bucket)`` where:
        - external_repo: ``''`` for ROCm/TheRock, or ``'{owner}-{repo}/'``
        - bucket: S3 bucket name
    """
    if not github_repository:
        github_repository = os.environ.get("GITHUB_REPOSITORY", "ROCm/TheRock")

    artifact_bucket_config = get_artifacts_bucket_config_for_workflow_run(
        github_repository=github_repository,
        release_type=release_type,
        workflow_run_id=workflow_run_id,
        workflow_run=workflow_run,
    )
    owner, repo_name = github_repository.split("/")
    external_repo = (
        f"{owner}-{repo_name}/"
        if artifact_bucket_config.name == "therock-ci-artifacts-external"
        else ""
    )
    return (external_repo, artifact_bucket_config.name)