TheRock/build_tools/github_actions/upload_tarballs.py

#!/usr/bin/env python3
# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

"""Upload tarballs to S3.

Uploads all .tar.gz files from the input directory to the tarballs/
subdirectory of the workflow output root in S3.

Example with ``--run-id 12345 --platform linux --release-type dev``:

    /tmp/tarballs/therock-dist-linux-gfx94X-dcgpu-7.10.0.tar.gz
      -> s3://therock-dev-artifacts/12345-linux/tarballs/therock-dist-linux-gfx94X-dcgpu-7.10.0.tar.gz

Usage:
    python build_tools/github_actions/upload_tarballs.py \\
        --input-tarballs-dir /tmp/tarballs \\
        --run-id 12345 --platform linux --release-type dev

    python build_tools/github_actions/upload_tarballs.py \\
        --input-tarballs-dir /tmp/tarballs \\
        --run-id 12345 --platform linux --release-type dev --dry-run

    # Local testing (no S3 credentials needed):
    python build_tools/github_actions/upload_tarballs.py \\
        --input-tarballs-dir /tmp/tarballs \\
        --run-id 12345 --platform linux \\
        --output-dir /tmp/upload-test
"""

import argparse
import json
import logging
import platform as platform_module
import sys
from pathlib import Path
from urllib.parse import quote

_BUILD_TOOLS_DIR = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_BUILD_TOOLS_DIR))

from _therock_utils.storage_backend import create_storage_backend
from _therock_utils.workflow_outputs import WorkflowOutputRoot
from github_actions_api import gha_set_output

logger = logging.getLogger(__name__)


def _tarball_url(output_root: WorkflowOutputRoot, name: str) -> str:
    return output_root.tarball(quote(name, safe="")).https_url


def _is_test_tarball(name: str) -> bool:
    return "-tests-" in name


def _select_multiarch_tarball_url(
    *,
    tarball_files: list[Path],
    output_root: WorkflowOutputRoot,
    platform: str,
) -> str | None:
    """Gets the default "multiarch" tarball URL from a list of tarball files.

    Note: this should look simpler once we drop the "multiarch" part of the
    file name, at which point the file will just be therock-dist-{platform}.
    """

    # Skip over "test" tarballs, only look at "base" tarballs.
    non_test_tarball_files = [f for f in tarball_files if not _is_test_tarball(f.name)]

    for f in non_test_tarball_files:
        name = f.name
        if name.startswith(f"therock-dist-{platform}-multiarch-"):
            return _tarball_url(output_root, name)

    return None


def run(
    input_tarballs_dir: Path,
    run_id: str,
    platform: str,
    release_type: str,
    output_dir: Path | None = None,
    dry_run: bool = False,
) -> int:
    tarballs_dir = input_tarballs_dir.resolve()
    if not tarballs_dir.is_dir():
        raise FileNotFoundError(f"Tarballs directory not found: {tarballs_dir}")

    tarball_files = sorted(tarballs_dir.glob("*.tar.gz"))
    if not tarball_files:
        raise FileNotFoundError(f"No .tar.gz files found in {tarballs_dir}")

    logger.info("Found %d tarballs in %s:", len(tarball_files), tarballs_dir)
    for f in tarball_files:
        size_mb = f.stat().st_size / (1024 * 1024)
        logger.info("  %s (%.1f MB)", f.name, size_mb)

    output_root = WorkflowOutputRoot.from_workflow_run(
        run_id=run_id,
        platform=platform,
        release_type=release_type or None,
    )

    dest = output_root.tarballs()
    logger.info("Destination: %s", dest.s3_uri)

    backend = create_storage_backend(
        staging_dir=output_dir,
        dry_run=dry_run,
    )
    count = backend.upload_directory(
        tarballs_dir,
        dest,
        include=["*.tar.gz"],
    )

    logger.info("Uploaded %d files", count)

    multiarch_tarball_url = _select_multiarch_tarball_url(
        tarball_files=tarball_files,
        output_root=output_root,
        platform=platform,
    )

    if not multiarch_tarball_url:
        raise ValueError(
            "No multiarch tarball URL was produced; check tarball naming and upload logic"
        )

    gha_set_output({"tarball_urls": json.dumps({"multiarch": multiarch_tarball_url})})

    return 0


def main(argv: list[str]) -> int:
    parser = argparse.ArgumentParser(description="Upload tarballs to S3")
    parser.add_argument(
        "--input-tarballs-dir",
        type=Path,
        required=True,
        help="Directory containing .tar.gz tarballs to upload",
    )
    parser.add_argument("--run-id", required=True, help="Workflow run ID")
    parser.add_argument(
        "--platform",
        default=platform_module.system().lower(),
        choices=["linux", "windows"],
        help="Platform (default: current system)",
    )
    parser.add_argument(
        "--release-type",
        default="ci",
        help='Release type: "ci", "dev", "nightly", or "prerelease"',
    )
    parser.add_argument(
        "--output-dir",
        type=Path,
        default=None,
        help="Output to local directory instead of S3 (for testing)",
    )
    parser.add_argument(
        "--dry-run", action="store_true", help="Print plan without uploading"
    )
    args = parser.parse_args(argv)

    return run(
        input_tarballs_dir=args.input_tarballs_dir,
        run_id=args.run_id,
        platform=args.platform,
        release_type=args.release_type,
        output_dir=args.output_dir,
        dry_run=args.dry_run,
    )


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    sys.exit(main(sys.argv[1:]))