TheRock/external-builds/uccl/build_prod_wheels.py

#!/usr/bin/env python
# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

r"""Builds production UCCL wheel based on the rocm wheels.

The UCCL project already has a TheRock build target that is currently
based on the prerelease (or nightly) wheels.

## Building interactively

A build consists of the two steps below:

1. Checkout repository:

The following command checks out the upstream repo into this directory,
which the script will use by default if they exist. Otherwise, checkout your
own and specify with `--uccl-dir` during the build step.

```
# On Linux, using default paths (nested under this folder):
python uccl_repo.py checkout
```

2. Build UCCL for a single gfx architecture.

Typical usage to build:

```
# On Linux, using the default path for the repository:
python build_prod_wheels.py \
    --output-dir $HOME/tmp/pyout \
    --index-url https://rocm.prereleases.amd.com/whl/gfx94X-dcgpu
```

## Building Linux portable wheels

UCCL's build process already produces a manylinux portable wheel. No
additional processing is required.
"""

import argparse
from datetime import date
import json
import os
from pathlib import Path
import platform
import re
import shutil
import shlex
import subprocess
import sys
import tempfile
import textwrap

script_dir = Path(__file__).resolve().parent

is_windows = platform.system() == "Windows"


def run_command(args: list[str | Path], cwd: Path, env: dict[str, str] | None = None):
    args = [str(arg) for arg in args]
    full_env = dict(os.environ)
    print(f"++ Exec [{cwd}]$ {shlex.join(args)}")
    if env:
        print(f":: Env:")
        for k, v in env.items():
            print(f"  {k}={v}")
        full_env.update(env)
    subprocess.check_call(args, cwd=str(cwd), env=full_env)


def copy_to_output(args: argparse.Namespace, src_file: Path):
    output_dir: Path = args.output_dir
    print(f"++ Copy {src_file} -> {output_dir}")
    output_dir.mkdir(parents=True, exist_ok=True)
    shutil.copy2(src_file, output_dir)


def directory_if_exists(dir: Path) -> Path | None:
    if dir.exists():
        return dir
    else:
        return None


def find_built_wheel(dist_dir: Path, dist_package: str) -> Path:
    dist_package = dist_package.replace("-", "_")
    glob = f"{dist_package}-*.whl"
    all_wheels = list(dist_dir.glob(glob))
    if not all_wheels:
        raise RuntimeError(f"No wheels matching '{glob}' found in {dist_dir}")
    if len(all_wheels) != 1:
        raise RuntimeError(f"Found multiple wheels matching '{glob}' in {dist_dir}")
    return all_wheels[0]


def do_build(args: argparse.Namespace):
    uccl_dir: Path | None = args.uccl_dir

    if is_windows:
        print("WARNING: UCCL does not builds on Windows.", file=sys.stderr)

    # Build UCCL
    if uccl_dir:
        run_command(
            [
                "./build.sh",
                "therock",
                "all",
                args.python_version,
                args.index_url,
                args.image,
            ],
            cwd=uccl_dir,
        )

        built_wheel = find_built_wheel(uccl_dir / "wheelhouse-therock", "uccl")
        print(f"Found built wheel: {built_wheel}")
        copy_to_output(args, built_wheel)
    else:
        print("--- Not building UCCL (no --uccl-dir)")


def main(argv: list[str]):
    p = argparse.ArgumentParser(prog="build_prod_wheels.py")

    p.add_argument(
        "--image",
        default="ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6",
        help="Base docker image for UCCL's build",
    )
    p.add_argument(
        "--output-dir",
        type=Path,
        required=True,
        help="Directory to copy built wheels to",
    )
    p.add_argument(
        "--uccl-dir",
        default=directory_if_exists(script_dir / "uccl"),
        type=Path,
        help="UCCL source directory",
    )
    p.add_argument(
        "--python-version",
        default=".".join(platform.python_version_tuple()[:2]),
        type=str,
        help="Python version to use for the build",
    )
    p.add_argument(
        "--index-url", required=True, help="Base URL of the Python Package Index."
    )

    args = p.parse_args(argv)
    do_build(args)


if __name__ == "__main__":
    main(sys.argv[1:])