# Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT """Utilities for producing Python packages.""" from typing import Callable, Sequence import importlib.util import io import json import re import os from pathlib import Path import platform import shlex import subprocess import shutil import sys import tarfile from .artifacts import ArtifactCatalog, ArtifactName from .exe_stub_gen import generate_exe_link_stub is_windows = platform.system() == "Windows" if not is_windows: # Used on Linux to check file types. Buggy/broken on Windows, but file # types are generally known from file extensions there. import magic BUILD_TOOLS_DIR = Path(__file__).resolve().parent.parent PYTHON_PACKAGING_DIR = BUILD_TOOLS_DIR / "packaging" / "python" / "templates" DIST_INFO_PATH = PYTHON_PACKAGING_DIR / "rocm" / "src" / "rocm_sdk" / "_dist_info.py" assert BUILD_TOOLS_DIR.exists() assert PYTHON_PACKAGING_DIR.exists() assert DIST_INFO_PATH.exists() ENABLED_VLOG_LEVEL: int = 5 def log(*args, vlog: int = 0, **kwargs): if vlog > ENABLED_VLOG_LEVEL: return file = sys.stdout print(*args, **kwargs, file=file) file.flush() class PopulatedFiles: """Tracks all populated files from the artifact catalog.""" def __init__(self): self.materialized_relpaths: dict[str, tuple["PopulatedDistPackage", Path]] = {} # Mapping of relpath of shared library aliases to the soname self.soname_aliases: dict[str, str] = {} def has(self, relpath: str) -> bool: return relpath in self.materialized_relpaths def mark_populated( self, package: "PopulatedDistPackage", relpath: str, dest_path: Path ): assert ( not relpath in self.materialized_relpaths ), f"File already populated {relpath}" self.materialized_relpaths[relpath] = (package, dest_path) class Parameters: """Stores all parameters needed for package generation.""" def __init__( self, dest_dir: Path, version: str, version_suffix: str, artifacts: ArtifactCatalog, kpack_split: bool = False, linux_target_families: list[str] | None = None, windows_target_families: list[str] | None = None, ): self.dest_dir = dest_dir self.version = version self.version_suffix = version_suffix self.artifacts = artifacts self.kpack_split = kpack_split self.all_target_families = artifacts.all_target_families # Cross-platform family view. When the multi-arch release pipeline # passes per-platform family lists, the rocm sdist must advertise # the union of both platforms' families and pick a # DEFAULT_TARGET_FAMILY that resolves on either OS so a user # without ROCM_SDK_TARGET_FAMILY / offload-arch lands on a family # that has wheels for their platform - prefer the intersection. # Without these kwargs the on-disk artifact view is used as # before, preserving today's behavior for single-platform builds. self.linux_target_families: list[str] = sorted(set(linux_target_families or [])) self.windows_target_families: list[str] = sorted( set(windows_target_families or []) ) cross_platform_inputs = ( linux_target_families is not None or windows_target_families is not None ) if cross_platform_inputs: linux_set = set(self.linux_target_families) windows_set = set(self.windows_target_families) self.available_target_families: list[str] = sorted(linux_set | windows_set) intersection = sorted(linux_set & windows_set) if intersection: self.default_target_family: str | None = intersection[0] elif self.available_target_families: self.default_target_family = self.available_target_families[0] else: self.default_target_family = None else: # Snapshot of the set; preserves prior set-iteration order so # the generated _dist_info.py is byte-identical to before for # single-platform builds. self.available_target_families = list(self.all_target_families) _sorted_families = sorted(self.all_target_families) self.default_target_family = ( _sorted_families[0] if _sorted_families else None ) self.populated_packages: list["PopulatedDistPackage"] = [] self.runtime_artifact_names: set[str] = set() # Load and interpolate the _dist_info.py template. # Base: version and nonce only — no family lines. Used as the starting # point for restrict_families packages so they can write clean family # content without a .clear() dance. dist_info_base = DIST_INFO_PATH.read_text() dist_info_base += f"__version__ = '{version}'\n" dist_info_base += f"PY_PACKAGE_SUFFIX_NONCE = '{version_suffix}'\n" self.dist_info_base_contents = dist_info_base # Full: base extended with all families. Used by most packages and by # the dynamically loaded self.dist_info module below. dist_info_contents = dist_info_base # In kpack-split mode, the libraries package is arch-neutral (no family # suffix). Override its dist_package_template so that # get_py_package_name(target_family=None) works for both the libraries # wheel itself and the device wheel overlay. if kpack_split: dist_info_contents += ( 'ALL_PACKAGES["libraries"].dist_package_template = ' '"rocm-sdk-libraries"\n' ) if self.default_target_family is not None: dist_info_contents += ( f"DEFAULT_TARGET_FAMILY = '{self.default_target_family}'\n" ) for target_family in self.available_target_families: dist_info_contents += ( f"AVAILABLE_TARGET_FAMILIES.append('{target_family}')\n" ) for target_family in self.linux_target_families: dist_info_contents += f"LINUX_TARGET_FAMILIES.append('{target_family}')\n" for target_family in self.windows_target_families: dist_info_contents += f"WINDOWS_TARGET_FAMILIES.append('{target_family}')\n" self.dist_info_contents = dist_info_contents # And dynamically load it so that we have access to the same config during # populate as will be loaded at setup and run time. spec = importlib.util.spec_from_loader("rocm_sdk_dist_info", loader=None) self.dist_info = importlib.util.module_from_spec(spec) exec(self.dist_info_contents, self.dist_info.__dict__) def filter_artifacts( self, filter: Callable[[ArtifactName], bool] = lambda _: True, includes: tuple[str] = (), excludes: tuple[str] = (), ) -> ArtifactCatalog: """Filters the global view of artifacts to only include a subset.""" return ArtifactCatalog( self.artifacts.artifact_dir, filter=filter, includes=includes, excludes=excludes, ) class PopulatedDistPackage: """Represents a single populated dist package, bootstrapped from a template.""" def __init__( self, params: Parameters, *, logical_name: str, target_family: str | None = None, restrict_families: bool = False, ): self.params = params self.logical_name = logical_name self.target_family = target_family try: self.entry = self.params.dist_info.ALL_PACKAGES[logical_name] except KeyError: raise KeyError( f"Logical package name {logical_name} not found (of {self.params.dist_info.ALL_PACKAGES.keys()})" ) self.rpath_deps: list[tuple["PopulatedDistPackage", str]] = [] self.files = PopulatedFiles() # restrict_families packages start from the base (no family lines) so # the per-family overrides below write clean content without a .clear() # dance. All other packages start from the full dist_info_contents. if restrict_families and target_family is not None: dist_info_contents = self.params.dist_info_base_contents else: dist_info_contents = self.params.dist_info_contents dist_info_contents += f"THIS_TARGET_FAMILY = {repr(target_family)}\n" dist_info_contents += ( f"THIS_PACKAGE_ENTRY = ALL_PACKAGES[{repr(logical_name)}]\n" ) # For per-family packages (e.g. meta/rocm in multi-arch builds), restrict # DEFAULT_TARGET_FAMILY and AVAILABLE_TARGET_FAMILIES to only this family # so that determine_target_family() at install time only resolves to this # family's packages. No .clear() needed — the base has an empty list. if restrict_families and target_family is not None: dist_info_contents += f"DEFAULT_TARGET_FAMILY = '{target_family}'\n" dist_info_contents += ( f"AVAILABLE_TARGET_FAMILIES.append('{target_family}')\n" ) # Device packages need to know the libraries package's Python package # name so their setup.py can set up the correct overlay directory. # The dist name ("rocm-sdk-libraries") is static and hardcoded in setup.py. if logical_name == "device": libraries_entry = self.params.dist_info.ALL_PACKAGES["libraries"] libraries_py_package_name = libraries_entry.get_py_package_name( target_family=None ) dist_info_contents += ( f"LIBRARIES_PY_PACKAGE_NAME = {repr(libraries_py_package_name)}\n" ) # Populate from template. self.path = self._copy_package_template( self.params.dest_dir, self.entry.template_directory, dist_info_relpath=f"src/{self.entry.pure_py_package_name}/_dist_info.py", dist_info_contents=dist_info_contents, dest_name=self.entry.get_dist_package_name(target_family=target_family), ) # Device packages overlay into the libraries package's platform dir # so .kpack files land alongside host .so files in site-packages. if logical_name == "device": libraries_entry = self.params.dist_info.ALL_PACKAGES["libraries"] self._platform_dir = ( self.path / "platform" / libraries_entry.get_py_package_name(target_family=None) ) else: self._platform_dir = ( self.path / "platform" / self.entry.get_py_package_name(self.target_family) ) @property def pure_dir(self) -> Path: """Gets the path of the pure package directory.""" return self.path / "src" / self.entry.pure_py_package_name @property def platform_dir(self) -> Path: """Directory that contains platform files. Created on access.""" self._platform_dir.mkdir(parents=True, exist_ok=True) (self._platform_dir / "__init__.py").touch() return self._platform_dir def rpath_dep( self, dep_package: "PopulatedDistPackage", rpath: str ) -> "PopulatedDistPackage": """Marks this package as needing to depend on RPATH from another package.""" self.rpath_deps.append((dep_package, rpath)) return self def _copy_package_template( self, dest_dir: Path, template_name: str, *, dist_info_relpath: str, dist_info_contents: str, dest_name: str | None = None, ) -> Path: """Copies a named template to the dest_dir and returns the Path. The template_name names a sub-directory of packaging/python. If no dest_name is specified, then a same-named sub-directory will be created in dest_dir and returned. Otherwise, it will be named dest_name. """ if dest_name is None: dest_name = template_name template_path = PYTHON_PACKAGING_DIR / template_name if not template_path.exists(): raise ValueError(f"Python package template {template_path} does not exist") package_dest_dir = dest_dir / dest_name if package_dest_dir.exists(): shutil.rmtree(package_dest_dir) log(f"::: Creating package '{template_name}': {package_dest_dir}") shutil.copytree( template_path, package_dest_dir, symlinks=True, dirs_exist_ok=True ) # Replace the _dist_info.py file. dist_info_path = package_dest_dir / dist_info_relpath log(f" Writing dist info: {dist_info_path}") dist_info_path.parent.mkdir(parents=True, exist_ok=True) dist_info_path.write_text(dist_info_contents, newline="\n") return package_dest_dir def populate_runtime_files( self, artifacts: ArtifactCatalog ) -> "PopulatedDistPackage": """Populates runtime files in an artifact catalog to the platform directory. All populated files are tracked in `files`, and any that were previously populated to another package are skipped. In this context, "runtime" means that we are only populating a subset of files needed by runtime packages. Specifically, this impacts shared library symlinks, which only populate the soname variant in the link farm. """ log( f"::: Populating runtime files {self.logical_name}[{self.target_family}]: " f"{self.path}" ) for an, an_path in artifacts.artifact_basedirs: log(f" + {an}: {an_path}") # Accumulate the artifact names we have created runtime packages for. # This will be used later to restrict devel packages to only these. self.params.runtime_artifact_names.add(an.name) package_dest_dir = self.platform_dir for relpath, dir_entry in artifacts.pm.matches(): if self.files.has(relpath): continue dest_path = package_dest_dir / relpath if dir_entry.is_symlink(): # Chase the symlink. self._populate_runtime_symlink(relpath, dest_path, dir_entry) else: # Copy the file. file_type = get_file_type(dir_entry) if file_type == "so": # We only populate runtime shared libraries that correspond # with their soname (or that don't have one). soname = get_soname(dir_entry.path) if soname: if soname == dir_entry.name: self._populate_file( relpath, dest_path, dir_entry, resolve_src=True ) else: self.files.soname_aliases[relpath] = soname continue # Otherwise, just copy the file. self._populate_file(relpath, dest_path, dir_entry, resolve_src=True) self.params.populated_packages.append(self) return self def populate_device_files( self, artifacts: ArtifactCatalog ) -> "PopulatedDistPackage": """Populates device files into the platform directory. Device artifacts are a mix of opaque binary data (.kpack archives, .co/.dat/.hsaco kernels, MIOpen DBs) and per-arch ELF shared libraries (e.g. libMIOpenCKGroupedConv_gfx1201.so). Data files are copied verbatim; ELF .so/exe files go through the same RPATH-patching path as populate_runtime_files() so their dynamic deps resolve across the kpack-split wheel layout. """ log( f"::: Populating device files {self.logical_name}[{self.target_family}]: " f"{self.path}" ) for an, an_path in artifacts.artifact_basedirs: log(f" + {an}: {an_path}") package_dest_dir = self.platform_dir libraries_py_package_name = package_dest_dir.name devel_links: list[dict[str, str]] = [] for relpath, dir_entry in artifacts.pm.matches(): if self.files.has(relpath): continue dest_path = package_dest_dir / relpath self._populate_file(relpath, dest_path, dir_entry, resolve_src=True) if not dir_entry.is_dir(): # Record the relative hardlink target into the libraries overlay # so that `rocm-sdk init` can mirror this per-ISA device file into # the generic rocm-sdk-devel tree. The backtrack count and target # layout mirror _populate_devel_file()'s symlink computation. backtrack = len(Path(relpath).parts) target = "/".join( [".."] * backtrack + [libraries_py_package_name, relpath] ) devel_links.append({"relpath": relpath, "target": target}) self._emit_devel_links_manifest(devel_links) self.params.populated_packages.append(self) return self def _emit_devel_links_manifest(self, devel_links: list[dict[str, str]]): """Writes the `_devel_links` manifest consumed by `rocm-sdk init`. Each device wheel ships this manifest so that, at devel expansion time, its per-ISA files are hardlinked from the rocm-sdk-libraries overlay into the generic rocm-sdk-devel tree (and recorded in this wheel's RECORD so that `pip uninstall` prunes them). Named per target family so that multiple device wheels overlay the shared libraries dir without colliding. """ manifest_dir = self.platform_dir / ".devel_links" manifest_dir.mkdir(parents=True, exist_ok=True) manifest_path = manifest_dir / f"{self.target_family}.json" manifest_path.write_text( json.dumps( {"version": self.params.version, "links": devel_links}, indent=2, ), newline="\n", ) def _populate_runtime_symlink( self, relpath: str, dest_path: Path, src_entry: os.DirEntry[str] ): # We can't have any symlinks in a runtime tree. # Here is what we do based on what it points to: # 1. Dangling or directory symlink: drop # 2. Shared library symlink: materialize if the symlink name is the SONAME # 3. Executable: Build a little executable launcher in place of the symlink # 4. Copy it into place (this should work for scripts and such -- hopefully). link_target = Path(src_entry.path).resolve() # Case 1: Drop dangling or directory symlink. if link_target.is_dir() or not link_target.exists(): return file_type = get_file_type(link_target) # Case 2: Shared library. if file_type == "so" and (soname := get_soname(link_target)): if soname == src_entry.name: self._populate_file(relpath, dest_path, src_entry, resolve_src=True) else: self.files.soname_aliases[relpath] = soname return # Case 3: Executable. if file_type == "exe": # Compile a standalone executable that dynamically emulates the symlink. raw_link_target = os.readlink(src_entry.path) log(f" EXESTUB: {relpath} (from {raw_link_target})", vlog=2) generate_exe_link_stub(dest_path, raw_link_target) self.files.mark_populated(self, relpath, dest_path) return # Case 4: Copy. self._populate_file(relpath, dest_path, src_entry, resolve_src=True) def _populate_file( self, relpath: str, dest_path: Path, src_entry: os.DirEntry[str], *, resolve_src: bool, ): is_symlink = src_entry.is_symlink() src_path = Path(src_entry.path) if resolve_src and is_symlink: src_path = src_path.resolve() dest_path.parent.mkdir(parents=True, exist_ok=True) # If it is a directory entry, just mkdir it. if src_entry.is_dir(): dest_path.mkdir(parents=False, exist_ok=True) return # It is a regular file of some kind. if dest_path.exists(): os.unlink(dest_path) # We have to patch many files, so we do not hard-link: always copy. log(f" MATERIALIZE: {relpath} (from {src_path})", vlog=2) shutil.copy2(src_path, dest_path) if self.files.has(relpath): log(f"WARNING: Path already materialized: {relpath}") else: self.files.mark_populated(self, relpath, dest_path) if not is_windows: # Update RPATHs on Linux. file_type = get_file_type(dest_path) if file_type == "exe" or file_type == "so": self._extend_rpath(dest_path) self._normalize_rpath(dest_path) def _extend_rpath(self, file_path: Path): for dep_project, rpath in self.rpath_deps: parent_relpath = self._platform_dir.parent.relative_to( file_path.parent, walk_up=True ) dep_py_package_name = dep_project.entry.get_py_package_name( self.target_family ) addl_rpath = f"$ORIGIN/{parent_relpath}/{dep_py_package_name}/{rpath}" log(f" ADD_RPATH: {file_path}: {addl_rpath}") patchelf_cl = [ "patchelf", "--add-rpath", addl_rpath, str(file_path), ] subprocess.check_call(patchelf_cl) def _normalize_rpath(self, file_path: Path): existing_rpath = ( subprocess.check_output( [ "patchelf", "--print-rpath", str(file_path), ] ) .decode() .strip() ) if not existing_rpath: return # Possibly in the future, do manual normalization of the RPATH. norm_rpath = existing_rpath log(f" NORMALIZE_RPATH: {file_path}: {norm_rpath}") subprocess.check_call( [ "patchelf", "--set-rpath", norm_rpath, # Forces the use of RPATH vs RUNPATH, which is more appropriate # for hermetic libraries like these since it does not allow # LD_LIBRARY_PATH to interfere. "--force-rpath", str(file_path), ] ) def populate_devel_files( self, *, addl_artifact_names: Sequence[str] = (), exclude_components: Sequence[str] = (), tarball_compression: bool = True, ): """Populates all files that have not yet been materialized and symlink the rest.""" package_path = self.platform_dir # Set up for what artifacts to include in the devel package, including # any emitted runtime artifacts plus additional requested. devel_artifact_names = set(self.params.runtime_artifact_names) devel_artifact_names.update(addl_artifact_names) # Exclude profiler-owned artifacts from the devel package. # # The profiler runtime (rocprofiler-compute and rocprofiler-systems) # is now packaged in the separate `rocm-profiler` wheel. However, # devel packaging automatically includes all runtime artifacts via # `runtime_artifact_names`, which would otherwise pull these profiler # artifacts back into the devel package. # # This leads to CI failures where devel tests attempt to load profiler # shared libraries without their full dependency closure (e.g. missing # rocprofiler-sdk or libomp resolution). # # Explicitly removing them here ensures correct package ownership: # - rocm-profiler → owns profiler runtime # - rocm-sdk-devel → does NOT include profiler runtime devel_artifact_names.difference_update( { "rocprofiler-compute", "rocprofiler-systems", } ) excluded = set(exclude_components) log(f":: Devel artifact inclusions: {devel_artifact_names}") def _devel_artifact_filter(an: ArtifactName) -> bool: if an.name not in devel_artifact_names: # We didn't generate a runtime artifact for it, so no devel # artifact. return False if an.component in excluded: return False if an.target_family != "generic" and an.target_family != self.target_family: # Only include artifacts for this devel package's target family. return False return True artifacts = self.params.filter_artifacts(_devel_artifact_filter) log(f"::: Populating devel package {package_path}") for an, an_path in artifacts.artifact_basedirs: log(f" + {an}: {an_path}") for relpath, dir_entry in artifacts.pm.matches(): dest_path = package_path / relpath self._populate_devel_file( relpath, dest_path, dir_entry, ) # For packaging, the devel platform/ contents are not wheel safe, so we # store them into their own tarball and dynamically decompress at runtime. # The tarball will contain as its first path component the top level # python package name that contains the platform files. tar_suffix = ".tar.xz" if tarball_compression else ".tar" tar_mode = "w:xz" if tarball_compression else "w" tar_path = self.pure_dir / f"_devel{tar_suffix}" log(f"::: Building secondary devel tarball: {tar_path}") with tarfile.open(tar_path, mode=tar_mode) as tf: for root, dirnames, files in os.walk(package_path): for file in list(files) + list(dirnames): file_path = os.path.join(root, file) arcname = os.path.relpath(file_path, package_path.parent) log(f"Adding {arcname}", vlog=2) tf.add(file_path, arcname=arcname, recursive=False) shutil.rmtree(package_path) def _find_populated( self, relpath: str ) -> "tuple[PopulatedDistPackage, Path] | None": """Search all populated runtime packages for a materialized relpath. When this devel package is target-family-specific, skip runtime packages that belong to a different (non-None) target family so that symlinks point into the correct arch's libraries package. """ for pkg in self.params.populated_packages: if not pkg.files.has(relpath): continue if ( self.target_family is not None and pkg.target_family is not None and pkg.target_family != self.target_family ): continue return pkg.files.materialized_relpaths[relpath] return None def _find_soname_alias(self, relpath: str) -> "str | None": """Search all populated runtime packages for a soname alias. When this devel package is target-family-specific, skip runtime packages that belong to a different (non-None) target family. """ for pkg in self.params.populated_packages: alias = pkg.files.soname_aliases.get(relpath) if alias is None: continue if ( self.target_family is not None and pkg.target_family is not None and pkg.target_family != self.target_family ): continue return alias return None def _populate_devel_file( self, relpath: str, dest_path: Path, src_entry: os.DirEntry[str] ): if src_entry.is_dir(follow_symlinks=False): dest_path.mkdir(parents=True, exist_ok=False) return # Re-add soname aliases. soname_alias = self._find_soname_alias(relpath) if soname_alias is not None: # This file is an alias to the proper soname. Just emit a relative # link. dest_path.symlink_to(soname_alias) return populated = self._find_populated(relpath) if populated is not None: # Already materialized: Link to it. populated_package, populated_path = populated # Materialize as a symlink to the original placement. This is tricky # because the symlink needs to be correct with respect to the install # placement, which is something like: # _rocm_sdk_core/path/to/foo.txt # _rocm_sdk_devel/path/to/foo.txt # In this example, relpath would be "path/to/foo.txt" and the proper # symlink: # ../../../_rocm_sdk_core/path/to/foo.txt # Conveniently, the root sibling is len(relpath_segments) up from the # original_path. # This is admittedly ugly, but there is not a super convenient way to # do it given that the relative path is computed for an installation # layout that is different from the packaging input layout. relpath_segment_count = len(Path(relpath).parts) backtrack_path = Path(*([".."] * relpath_segment_count)) link_target = backtrack_path / populated_package.platform_dir.name / relpath log(f"DEVLINK: {relpath} -> {link_target}", vlog=2) if dest_path.exists(follow_symlinks=False): dest_path.unlink() dest_path.parent.mkdir(parents=True, exist_ok=True) dest_path.symlink_to(link_target) return # If the source is a symlink, faithfully transcribe it. if src_entry.is_symlink(): if dest_path.exists(follow_symlinks=False): dest_path.unlink() target_path = os.readlink(src_entry.path) log(f"LINK: {relpath} (to {target_path})", vlog=2) os.symlink(target_path, dest_path) return # Otherwise, no one else has emitted it, so just materialize verbatim. log(f"MATERIALIZE: {relpath} (from {src_entry.path})", vlog=2) if dest_path.exists(follow_symlinks=False): dest_path.unlink() dest_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src_entry.path, dest_path) if not is_windows: # Update RPATHs on Linux. file_type = get_file_type(dest_path) if file_type == "exe" or file_type == "so": self._normalize_rpath(dest_path) MAGIC_AR_MATCH = re.compile("ar archive") MAGIC_EXECUTABLE_MATCH = re.compile("ELF[^,]+executable,") MAGIC_SO_MATCH = re.compile("ELF[^,]+shared object,") def get_file_type(dir_entry: os.DirEntry[str] | Path) -> str: if isinstance(dir_entry, os.DirEntry): path = Path(dir_entry.path) else: path = Path(dir_entry) if path.is_symlink(): return "symlink" if path.is_dir(): return "dir" # We only care about finding certain needles in the haystack, so exclude # text-like files from the get-go. path = str(path) if path.endswith(".txt") or path.endswith(".h") or path.endswith(".hpp"): return "text" elif path.endswith(".hsaco") or path.endswith(".co"): # These read as shared libraries. return "hsaco" elif path.endswith(".lib"): return "ar" elif path.endswith(".exe"): return "exe" if is_windows: # Don't try to use 'magic' on Windows, since it is buggy/broken. # Hopefully the file type was covered by an extension check above. return "other" desc = magic.from_file(path) if MAGIC_EXECUTABLE_MATCH.search(desc): return "exe" if MAGIC_SO_MATCH.search(desc): return "so" if MAGIC_AR_MATCH.search(desc): return "ar" return "other" def get_soname(sofile: Path) -> str: return ( subprocess.check_output(["patchelf", "--print-soname", str(sofile)]) .decode() .strip() ) def build_packages( dest_dir: Path, *, wheel_compression: bool = True, package_dirs: list[Path] | None = None, dist_dir: Path | None = None, ): effective_dist_dir = dist_dir or (dest_dir / "dist") effective_dist_dir.mkdir(parents=True, exist_ok=True) if package_dirs is None: package_dirs = [ p for p in dest_dir.iterdir() if p.is_dir() and (p / "pyproject.toml").exists() ] for child_path in package_dirs: child_name = child_path.name # Some of our packages build as sdists and some as wheels. # Contrary to documented wisdom, we invoke setuptools directly. This is # because the "build frontends" have an impossible compatibility matrix # and opinions about how to pass arguments to the backends. So we skip # the frontends for such a closed case as this. setuppy_path = child_path / "setup.py" build_args = [ sys.executable, str(setuppy_path.resolve()), ] if child_name in ["rocm"]: build_args.append("sdist") else: build_args.append("bdist_wheel") if not wheel_compression: build_args.append("--compression") build_args.append("stored") build_args.extend( [ "-v", "--dist-dir", str(effective_dist_dir.resolve()), ] ) log(f"::: Building python package {child_name}: {shlex.join(build_args)}") subprocess.check_call(build_args, cwd=child_path, stderr=subprocess.STDOUT) if child_name == "rocm": # setuptools writes PKG-INFO / setup.cfg / egg-info in platform # text mode, which yields CRLF on Windows. Normalize so the # rocm sdist's text members are content-identical across Linux # and Windows builds (tar metadata such as mtime may still # differ). sdists = list(effective_dist_dir.glob(f"{child_name}-*.tar.gz")) if not sdists: raise RuntimeError( f"No {child_name} sdist produced in {effective_dist_dir}" ) for sdist in sdists: _normalize_sdist_line_endings(sdist) _TEXT_SUFFIXES = frozenset({".py", ".md", ".toml", ".cfg", ".txt", ".rst", ".in"}) def _is_text_member(name: str) -> bool: base = name.rsplit("/", 1)[-1] if base == "PKG-INFO": return True suffix = ("." + base.rsplit(".", 1)[-1].lower()) if "." in base else "" return suffix in _TEXT_SUFFIXES def _normalize_sdist_line_endings(sdist_path: Path) -> None: """Rewrite `sdist_path` in place with LF line endings on text members. Preserves the original tar member order and per-file metadata (mtime, mode, uid/gid) so the only delta vs. setuptools' output is the byte content of text files. """ with tarfile.open(sdist_path, "r:gz") as src: members = src.getmembers() payloads: dict[str, bytes] = {} for member in members: if not member.isfile(): continue fp = src.extractfile(member) if fp is None: raise RuntimeError( f"Could not extract regular file {member.name} from {sdist_path}" ) data = fp.read() if _is_text_member(member.name): data = data.replace(b"\r\n", b"\n") payloads[member.name] = data tmp_path = sdist_path.with_suffix(sdist_path.suffix + ".tmp") with tarfile.open(tmp_path, "w:gz") as dst: for member in members: if member.isfile() and member.name in payloads: data = payloads[member.name] member.size = len(data) dst.addfile(member, fileobj=io.BytesIO(data)) else: dst.addfile(member) tmp_path.replace(sdist_path) if not sdist_path.exists() or sdist_path.stat().st_size == 0: raise RuntimeError(f"Normalized sdist missing or empty: {sdist_path}")