#!/usr/bin/env python3 # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT """ Packaging + repository upload tool. Usage (multi-arch CI — new-style, preferred): python ./build_tools/packaging/linux/upload_package_repo.py \ --pkg-type deb \ --run-id 16418185899 Bucket + prefix are resolved automatically via WorkflowOutputRoot using the GITHUB_REPOSITORY and RELEASE_TYPE environment variables: - CI builds → therock-ci-artifacts / -linux/packages/deb - dev builds → therock-dev-artifacts / -linux/packages/deb - nightly → therock-nightly-artifacts / -linux/packages/deb - prerelease → therock-prerelease-artifacts / -linux/packages/deb Usage (single-arch release — legacy, build_native_linux_packages.yml): python ./build_tools/packaging/linux/upload_package_repo.py \ --pkg-type deb \ --s3-bucket therock-nightly-packages \ --amdgpu-family gfx94X-dcgpu \ --artifact-id 16418185899 \ --job nightly Legacy upload locations: dev/nightly → s3bucket//- prerelease → s3bucket/v3/packages/ """ import argparse import boto3 import datetime import os import shutil import subprocess import sys from pathlib import Path _THIS_DIR = Path(__file__).resolve().parent _BUILD_TOOLS_DIR = _THIS_DIR.parent.parent if str(_THIS_DIR) not in sys.path: sys.path.insert(0, str(_THIS_DIR)) if str(_BUILD_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_BUILD_TOOLS_DIR)) from _therock_utils.storage_location import StorageLocation from _therock_utils.workflow_outputs import WorkflowOutputRoot def regenerate_rpm_metadata_from_s3(s3, bucket, prefix, uploaded_packages): """Regenerate RPM repository metadata using merge approach. Downloads existing repodata from S3, generates metadata for new packages, merges them using mergerepo_c, and uploads the result back to S3. Args: s3: boto3 S3 client bucket: S3 bucket name prefix: S3 prefix (e.g., 'rpm/20251222-12345') uploaded_packages: List of actually uploaded .rpm file paths """ import tempfile print(f"Updating RPM repository metadata (merge mode)...") # Create temporary directory for metadata operations with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) # Efficient approach: Download existing repodata and merge with new packages old_repo_dir = temp_path / "old_repo" new_repo_dir = temp_path / "new_repo" merged_repo_dir = temp_path / "merged_repo" old_repo_dir.mkdir(parents=True, exist_ok=True) new_repo_dir.mkdir(parents=True, exist_ok=True) merged_repo_dir.mkdir(parents=True, exist_ok=True) # Step 1: Download existing repodata from S3 (small files) old_repodata_dir = old_repo_dir / "repodata" old_repodata_dir.mkdir(parents=True, exist_ok=True) print( f"Downloading existing repository metadata from S3: s3://{bucket}/{prefix}/x86_64/repodata/" ) repodata_files = [] try: paginator = s3.get_paginator("list_objects_v2") for page in paginator.paginate( Bucket=bucket, Prefix=f"{prefix}/x86_64/repodata/" ): if "Contents" not in page: continue for obj in page["Contents"]: key = obj["Key"] filename = Path(key).name local_file = old_repodata_dir / filename s3.download_file(bucket, key, str(local_file)) repodata_files.append(filename) print(f" Downloaded: {filename}") if repodata_files: print( f"✅ Found {len(repodata_files)} existing metadata files to merge" ) else: print("No existing metadata files found") except Exception as e: print(f"⚠️ No existing repodata found (new repo?): {e}") # Step 2: Generate repodata for NEW packages only (actually uploaded ones) rpm_packages = [p for p in uploaded_packages if p.endswith(".rpm")] if rpm_packages: print( f"Generating metadata for {len(rpm_packages)} uploaded RPM packages..." ) # Copy uploaded RPMs to temp dir new_arch_dir = new_repo_dir / "x86_64" new_arch_dir.mkdir(parents=True, exist_ok=True) for rpm_file in rpm_packages: shutil.copy2(rpm_file, new_arch_dir / Path(rpm_file).name) # Generate repodata for new packages with clean paths (no baseurl) run_command( ["createrepo_c", "--no-database", "--simple-md-filenames", "."], cwd=str(new_arch_dir), ) print("✅ Generated metadata for uploaded packages") else: print("No new RPM packages uploaded (all deduplicated)") # Still need to ensure old metadata is preserved! if repodata_files: print("Preserving existing repodata...") # Just re-upload the existing repodata we downloaded for metadata_file in old_repodata_dir.iterdir(): if metadata_file.is_file(): s3_key = f"{prefix}/x86_64/repodata/{metadata_file.name}" s3.upload_file(str(metadata_file), bucket, s3_key) print(f" Uploaded: {metadata_file.name}") print("✅ RPM repository metadata preserved") return # Step 3: Merge repositories using mergerepo_c (no need to download all RPMs!) merged_arch_dir = merged_repo_dir / "x86_64" merged_arch_dir.mkdir(parents=True, exist_ok=True) if repodata_files: # If we have existing metadata print("Merging old and new repository metadata...") # mergerepo_c merges repodata without needing actual RPM files! # Use --no-database, --simple-md-filenames, and --omit-baseurl to ensure clean paths run_command( [ "mergerepo_c", "--no-database", "--simple-md-filenames", "--omit-baseurl", "--repo", str(old_repo_dir), "--repo", str(new_repo_dir / "x86_64"), "--outputdir", str(merged_arch_dir), ], cwd=str(temp_path), ) print("✅ Merged repository metadata") else: # First upload, no existing metadata print("First upload - using new repository metadata") shutil.copytree( new_repo_dir / "x86_64" / "repodata", merged_arch_dir / "repodata" ) # Step 4: Upload merged repodata to S3 merged_repodata = merged_arch_dir / "repodata" if merged_repodata.exists(): print("Uploading merged repository metadata to S3...") uploaded_metadata = [] for metadata_file in merged_repodata.iterdir(): if metadata_file.is_file(): s3_key = f"{prefix}/x86_64/repodata/{metadata_file.name}" s3.upload_file(str(metadata_file), bucket, s3_key) uploaded_metadata.append(metadata_file.name) print(f" Uploaded: {metadata_file.name}") print(f"✅ RPM repository metadata updated: {len(uploaded_metadata)} files") def generate_release_file_with_checksums(release_file, job_type, dists_dir): """Generate a Debian Release file with MD5Sum, SHA1, and SHA256 checksums. Args: release_file: Path to the Release file to create job_type: Job type for metadata (nightly/dev/release) dists_dir: Directory containing Packages files (main/binary-amd64/) """ import hashlib # Files to hash (relative paths from dists/stable/) files_to_hash = [ (dists_dir / "Packages", "main/binary-amd64/Packages"), (dists_dir / "Packages.gz", "main/binary-amd64/Packages.gz"), ] # Calculate all hashes md5_entries = [] sha1_entries = [] sha256_entries = [] for file_path, rel_path in files_to_hash: if not file_path.exists(): continue # Get file size file_size = file_path.stat().st_size # Calculate hashes md5_hash = hashlib.md5() sha1_hash = hashlib.sha1() sha256_hash = hashlib.sha256() with open(file_path, "rb") as f: while True: data = f.read(65536) # Read in 64KB chunks if not data: break md5_hash.update(data) sha1_hash.update(data) sha256_hash.update(data) # Store entries (space-aligned format) md5_entries.append(f" {md5_hash.hexdigest()} {file_size:16d} {rel_path}") sha1_entries.append(f" {sha1_hash.hexdigest()} {file_size:16d} {rel_path}") sha256_entries.append(f" {sha256_hash.hexdigest()} {file_size:16d} {rel_path}") # Write Release file with open(release_file, "w", encoding="utf-8") as f: # Header fields f.write( f"""Origin: AMD ROCm Label: ROCm {job_type} Packages Suite: stable Codename: stable Architectures: amd64 Components: main Description: ROCm APT Repository Date: {datetime.datetime.utcnow():%a, %d %b %Y %H:%M:%S UTC} """ ) # MD5Sum section if md5_entries: f.write("MD5Sum:\n") f.write("\n".join(md5_entries)) f.write("\n") # SHA1 section if sha1_entries: f.write("SHA1:\n") f.write("\n".join(sha1_entries)) f.write("\n") # SHA256 section if sha256_entries: f.write("SHA256:\n") f.write("\n".join(sha256_entries)) f.write("\n") print(f"✅ Release file generated with checksums: MD5, SHA1, SHA256") def upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file): """Helper function to upload Debian metadata files to S3. Args: s3: boto3 S3 client bucket: S3 bucket name prefix: S3 prefix dists_dir: Directory containing Packages files release_file: Path to Release file """ packages_file = dists_dir / "Packages" packages_gz = dists_dir / "Packages.gz" uploaded_count = 0 if packages_file.exists(): s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" s3.upload_file(str(packages_file), bucket, s3_key) print(f" Uploaded: Packages") uploaded_count += 1 if packages_gz.exists(): s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages.gz" s3.upload_file(str(packages_gz), bucket, s3_key) print(f" Uploaded: Packages.gz") uploaded_count += 1 if release_file.exists(): s3_key = f"{prefix}/dists/stable/Release" s3.upload_file(str(release_file), bucket, s3_key) print(f" Uploaded: Release") uploaded_count += 1 print(f"✅ DEB repository metadata updated: {uploaded_count} files") def regenerate_deb_metadata_from_s3( s3, bucket, prefix, uploaded_packages, job_type="nightly" ): """Regenerate Debian repository metadata efficiently with proper checksums. Uses dpkg-scanpackages for efficiency (no package downloads), but generates proper Release file with MD5Sum, SHA1, and SHA256 checksums. Args: s3: boto3 S3 client bucket: S3 bucket name prefix: S3 prefix (e.g., 'deb/20251222-12345') uploaded_packages: List of actually uploaded .deb file paths job_type: Job type for Release file metadata (default: 'nightly') """ import tempfile print(f"Updating DEB repository metadata (merge mode with checksums)...") with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) # Setup directories dists_dir = temp_path / "dists" / "stable" / "main" / "binary-amd64" dists_dir.mkdir(parents=True, exist_ok=True) pool_dir = temp_path / "pool" / "main" pool_dir.mkdir(parents=True, exist_ok=True) # Step 1: Download existing Packages file from S3 (SMALL FILE - efficient!) existing_packages = dists_dir / "Packages.old" packages_s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" try: print( f"Downloading existing Packages file from S3: s3://{bucket}/{packages_s3_key}" ) s3.download_file(bucket, packages_s3_key, str(existing_packages)) with open(existing_packages, "r", encoding="utf-8") as f: content = f.read() pkg_count = content.count("\nPackage: ") print(f"✅ Downloaded existing Packages file ({pkg_count} packages)") except Exception as e: print(f"⚠️ No existing Packages file found (new repo?): {e}") existing_packages = None # Step 2: Generate Packages entries for NEW packages only deb_packages = [p for p in uploaded_packages if p.endswith(".deb")] if deb_packages: print( f"Generating Packages entries for {len(deb_packages)} uploaded DEB packages..." ) # Copy uploaded DEBs to temp dir for deb_file in deb_packages: shutil.copy2(deb_file, pool_dir / Path(deb_file).name) # Generate Packages entries for uploaded packages new_packages = dists_dir / "Packages.new" run_command( ["dpkg-scanpackages", "-m", "pool/main", "/dev/null"], cwd=str(temp_path), stdout=new_packages, ) print("✅ Generated Packages entries for uploaded packages") else: print("No new DEB packages uploaded (all deduplicated)") if existing_packages and existing_packages.exists(): print("Preserving existing metadata...") shutil.copy2(existing_packages, dists_dir / "Packages") run_command( ["gzip", "-9c", "Packages"], cwd=str(dists_dir), stdout=dists_dir / "Packages.gz", ) # Generate Release file with checksums release_dir = temp_path / "dists" / "stable" release_dir.mkdir(parents=True, exist_ok=True) release_file = release_dir / "Release" generate_release_file_with_checksums(release_file, job_type, dists_dir) # Upload preserved files upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file) return # Step 3: Merge old and new Packages files merged_packages = dists_dir / "Packages" if existing_packages and existing_packages.exists(): print("Merging old and new Packages files...") def parse_packages_file(filepath): """Parse Packages file into dict keyed by Filename""" packages = {} with open(filepath, "r", encoding="utf-8") as f: current_entry = [] current_filename = None for line in f: if line.strip() == "": if current_entry and current_filename: packages[current_filename] = ( "\n".join(current_entry) + "\n" ) current_entry = [] current_filename = None else: current_entry.append(line.rstrip()) if line.startswith("Filename:"): current_filename = line.split(":", 1)[1].strip() if current_entry and current_filename: packages[current_filename] = "\n".join(current_entry) + "\n" return packages old_packages = parse_packages_file(existing_packages) new_packages_dict = parse_packages_file(new_packages) print(f" Old metadata: {len(old_packages)} packages") print(f" New metadata: {len(new_packages_dict)} packages") merged = old_packages.copy() merged.update(new_packages_dict) with open(merged_packages, "w", encoding="utf-8") as outfile: for filename in sorted(merged.keys()): outfile.write(merged[filename]) outfile.write("\n") print(f"✅ Merged Packages files: {len(merged)} total packages") else: print("First upload - using new Packages file") shutil.copy2(new_packages, merged_packages) # Compress Packages file run_command( ["gzip", "-9c", "Packages"], cwd=str(dists_dir), stdout=dists_dir / "Packages.gz", ) # Step 4: Generate Release file with checksums release_dir = temp_path / "dists" / "stable" release_dir.mkdir(parents=True, exist_ok=True) release_file = release_dir / "Release" generate_release_file_with_checksums(release_file, job_type, dists_dir) # Step 5: Upload merged files to S3 upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file) def regenerate_repo_metadata_from_s3( s3, bucket, prefix, pkg_type, uploaded_packages, job_type="nightly" ): """Regenerate repository metadata efficiently using merge approach. This uses mergerepo_c (RPM) or merges Packages files (DEB) to efficiently update metadata without re-downloading all packages from S3. Args: s3: boto3 S3 client bucket: S3 bucket name prefix: S3 prefix (e.g., 'rpm/20251222-12345') pkg_type: Package type ('rpm' or 'deb') uploaded_packages: List of actually uploaded package file paths (avoids duplicates from deduplication) job_type: Job type for Release file metadata (default: 'nightly') """ if pkg_type == "rpm": regenerate_rpm_metadata_from_s3(s3, bucket, prefix, uploaded_packages) elif pkg_type == "deb": regenerate_deb_metadata_from_s3(s3, bucket, prefix, uploaded_packages, job_type) else: raise ValueError(f"Unsupported package type: {pkg_type}") def run_command(cmd: list[str], cwd=None, stdout=None): """Run a command safely without shell interpolation. Args: cmd: Command and arguments as a list of strings cwd: Working directory for the command stdout: Optional path to redirect stdout to a file Raises: subprocess.CalledProcessError: If the command exits with non-zero status """ print(f"Running: {' '.join(cmd)}") if stdout is not None: with open(stdout, "wb") as f: subprocess.run(cmd, check=True, cwd=cwd, stdout=f) else: subprocess.run(cmd, check=True, cwd=cwd) def find_package_dir(): base = Path.cwd() / "output" / "packages" if not base.exists(): raise RuntimeError(f"Package directory not found: {base}") return base def yyyymmdd(): return datetime.datetime.utcnow().strftime("%Y%m%d") def s3_object_exists(s3, bucket, key): try: s3.head_object(Bucket=bucket, Key=key) return True except s3.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": return False raise def create_deb_repo(package_dir, job_type): print("Creating APT repository...") package_path = Path(package_dir) dists = package_path / "dists" / "stable" / "main" / "binary-amd64" pool = package_path / "pool" / "main" dists.mkdir(parents=True, exist_ok=True) pool.mkdir(parents=True, exist_ok=True) for f in package_path.iterdir(): if f.suffix == ".deb": shutil.move(f, pool / f.name) run_command( ["dpkg-scanpackages", "-m", "pool/main", "/dev/null"], cwd=str(package_path), stdout=dists / "Packages", ) run_command( ["gzip", "-9c", "Packages"], cwd=str(dists), stdout=dists / "Packages.gz", ) release = package_path / "dists" / "stable" / "Release" with open(release, "w", encoding="utf-8") as f: f.write( f"""Origin: AMD ROCm Label: ROCm {job_type} Packages Suite: stable Codename: stable Architectures: amd64 Components: main Date: {datetime.datetime.utcnow():%a, %d %b %Y %H:%M:%S UTC} """ ) def create_rpm_repo(package_dir): """Create RPM repository structure. Note: Repository metadata (repodata) will be regenerated from S3 after upload to ensure it reflects all packages, including deduplicated ones. """ print("Creating RPM repository...") package_path = Path(package_dir) arch_dir = package_path / "x86_64" arch_dir.mkdir(parents=True, exist_ok=True) for f in package_path.iterdir(): if f.suffix == ".rpm": shutil.move(f, arch_dir / f.name) # Generate initial repodata from local packages with clean paths (no baseurl) # This will be regenerated from S3 state after upload run_command( ["createrepo_c", "--no-database", "--simple-md-filenames", "."], cwd=str(arch_dir), ) def upload_to_s3(source_dir, bucket, prefix, dedupe=False): s3 = boto3.client("s3") print(f"Uploading to s3://{bucket}/{prefix}/") print(f"Deduplication: {'ON' if dedupe else 'OFF'}") skipped = 0 uploaded = 0 uploaded_packages = [] # Track actually uploaded package files for root, _, files in os.walk(source_dir): for fname in files: # Always skip local index.html files, those are generated server-side. if fname == "index.html": continue # Skip build manifest files - these are for local tracking only if fname.lower().endswith(".txt"): print(f"Skipping build manifest file (local only): {fname}") continue local = Path(root) / fname rel = local.relative_to(source_dir) key = f"{prefix}/{rel.as_posix()}" # Skip metadata files - they'll be regenerated/merged properly later # For DEB: skip Packages, Packages.gz, Release in dists/ # For RPM: skip repodata/* files if "/repodata/" in key or key.endswith("/repodata"): print(f"Skipping metadata file (will regenerate): {fname}") continue if "/dists/" in key and ( fname in ["Packages", "Packages.gz", "Release", "InRelease"] ): print(f"Skipping metadata file (will regenerate): {fname}") continue if dedupe and (fname.endswith(".deb") or fname.endswith(".rpm")): if s3_object_exists(s3, bucket, key): print(f"Skipping existing package: {fname}") skipped += 1 continue extra = {"ContentType": "text/html"} if fname.endswith(".html") else None print(f"Uploading: {key}") s3.upload_file(str(local), bucket, key, ExtraArgs=extra) uploaded += 1 # Track uploaded packages for metadata generation if fname.endswith(".deb") or fname.endswith(".rpm"): uploaded_packages.append(str(local)) print(f"Uploaded: {uploaded}, Skipped: {skipped}") if uploaded_packages: print(f"Uploaded packages: {[Path(p).name for p in uploaded_packages]}") return s3, uploaded_packages # Return S3 client and list of uploaded packages def _emit_github_output(key: str, value: str) -> None: """Write a key=value pair to $GITHUB_OUTPUT if running in GitHub Actions.""" github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with open(github_output, "a", encoding="utf-8") as f: f.write(f"{key}={value}\n") def _package_install_url(bucket: str, prefix: str, pkg_type: str) -> str: """Compute the package manager install URL for a given repo location. For RPM repos, dnf/yum baseurl must point to the x86_64/ subdirectory (the directory containing repodata/). For DEB repos, apt points to the repo root (it resolves dists/ itself). """ base = StorageLocation(bucket, prefix).https_url if pkg_type == "rpm": return f"{base}/x86_64" return base def _resolve_upload_target( args: argparse.Namespace, pkg_type: str, ) -> tuple[str, str, str, bool, str]: """Resolve S3 bucket, prefix, install URL, dedupe flag, and job type. Returns: Tuple of (bucket, prefix, install_url, dedupe, job_type) """ if args.run_id: # New-style: derive bucket + prefix from WorkflowOutputRoot. # This is the single source of truth for CI path layout. root = WorkflowOutputRoot.from_workflow_run( run_id=args.run_id, platform="linux" ) loc = root.native_linux_packages(pkg_type) job_type = os.environ.get("RELEASE_TYPE", "ci") install_url = _package_install_url(loc.bucket, loc.relative_path, pkg_type) return loc.bucket, loc.relative_path, install_url, True, job_type if args.s3_prefix: # Legacy: explicit prefix provided by get_s3_config.py return ( args.s3_bucket, args.s3_prefix, _package_install_url(args.s3_bucket, args.s3_prefix, pkg_type), True, args.job, ) if args.job in ("nightly", "dev"): prefix = f"{pkg_type}/{yyyymmdd()}-{args.artifact_id}" return ( args.s3_bucket, prefix, _package_install_url(args.s3_bucket, prefix, pkg_type), True, args.job, ) if args.job == "prerelease": prefix = f"v3/packages/{pkg_type}" return ( args.s3_bucket, prefix, _package_install_url(args.s3_bucket, prefix, pkg_type), True, args.job, ) raise ValueError(f"Unknown job type: {args.job!r}") def main(): parser = argparse.ArgumentParser() parser.add_argument("--pkg-type", required=True, choices=["deb", "rpm"]) # New-style args: use WorkflowOutputRoot for bucket/prefix resolution. # When --run-id is provided, bucket and prefix are derived automatically # from CI context (GITHUB_REPOSITORY, RELEASE_TYPE, fork detection). parser.add_argument( "--run-id", help="GitHub Actions workflow run ID (enables WorkflowOutputRoot path resolution)", ) # Legacy args: kept for backward compatibility with build_native_linux_packages.yml parser.add_argument("--s3-bucket") parser.add_argument("--amdgpu-family", required=False) # unused, kept for compat parser.add_argument("--artifact-id") parser.add_argument( "--job", default="dev", choices=["dev", "nightly", "prerelease"], help="Job type (legacy, used when --run-id is not provided)", ) parser.add_argument( "--s3-prefix", required=False, help="Override S3 prefix (legacy, used when --run-id is not provided)", ) parser.add_argument( "--package-dir", required=True, help="Path to the directory containing built packages.", ) args = parser.parse_args() package_dir = Path(args.package_dir).resolve() bucket, prefix, install_url, dedupe, job_type = _resolve_upload_target( args, args.pkg_type ) if args.pkg_type == "deb": create_deb_repo(package_dir, job_type) else: create_rpm_repo(package_dir) # Upload packages and metadata to S3 s3_client, uploaded_packages = upload_to_s3( package_dir, bucket, prefix, dedupe=dedupe ) # Efficiently update repository metadata by merging with existing metadata regenerate_repo_metadata_from_s3( s3_client, bucket, prefix, args.pkg_type, uploaded_packages, job_type ) print(f"Package repository URL: {install_url}") _emit_github_output("package_repository_url", install_url) if __name__ == "__main__": main()