TheRock/.github/workflows/multi_arch_ci_linux.yml

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

name: Multi-Arch CI - Linux

on:
  workflow_call:
    inputs:
      build_config:
        type: string
        description: >-
          JSON object with build configuration for this platform. Fields:
          artifact_group, per_family_info, dist_amdgpu_families,
          build_variant_label, build_variant_cmake_preset,
          build_variant_suffix, build_pytorch, build_jax,
          build_native_linux, test_python_packages_matrix,
          pytorch_build_matrix, jax_build_matrix,
          prebuilt_stages, baseline_run_id.
      test_labels:
        type: string
      rocm_package_version:
        type: string
      rocm_deb_package_version:
        type: string
        description: ROCm DEB package version for native packages
      rocm_rpm_package_version:
        type: string
        description: ROCm RPM package version for native packages
      test_type:
        type: string
      release_type:
        description: 'Release type: "ci", "dev", "nightly", or "prerelease"'
        type: string
        default: ci
      external_repo_config:
        description: "JSON config for external repo checkout (repository, ref, checkout_path, source_package, fetch_sources_args)"
        type: string
        default: ""
      changed_projects:
        description: 'Comma-separated list of projects to test (e.g., "rocblas,hipblas" or "projects/rocblas"). Empty = run all tests.'
        type: string
        default: ""
      repository:
        description: "Repository to checkout. Defaults to github.repository."
        type: string
        default: ""
      ref:
        description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref."
        type: string
        default: ""

permissions:
  contents: read

jobs:
  copy_prebuilt_stages:
    name: Copy Prebuilt Stages
    if: ${{ fromJSON(inputs.build_config).prebuilt_stages != '' && fromJSON(inputs.build_config).baseline_run_id != '' }}
    runs-on: aws-linux-scale-rocm-prod-rc
    permissions:
      contents: read
      id-token: write
    container:
      image: python:3.12-slim
    steps:
      - name: Checking out repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ inputs.repository || github.repository }}
          ref: ${{ inputs.ref }}

      - name: Install artifact manager dependencies
        run: pip install boto3

      - name: Configure AWS credentials for artifact uploads
        uses: ./.github/actions/configure_aws_artifacts_credentials
        with:
          release_type: ${{ inputs.release_type }}

      - name: Copy prebuilt stage artifacts
        env:
          GITHUB_TOKEN: ${{ github.token }}
          RELEASE_TYPE: ${{ inputs.release_type }}
        run: |
          python build_tools/artifact_manager.py copy \
            --source-run-id=${{ fromJSON(inputs.build_config).baseline_run_id }} \
            --source-repository=${{ fromJSON(inputs.build_config).baseline_repository }} \
            --stage="${{ fromJSON(inputs.build_config).prebuilt_stages }}" \
            --amdgpu-families="${{ fromJSON(inputs.build_config).dist_amdgpu_families }}" \
            --expand-family-to-targets

  build_multi_arch_stages:
    name: Build Multi-Arch Stages
    needs: copy_prebuilt_stages
    if: ${{ !cancelled() && !failure() }}
    uses: ./.github/workflows/multi_arch_build_portable_linux.yml
    secrets: inherit
    with:
      matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }}
      dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
      build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }}
      build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }}
      build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }}
      prebuilt_stages: ${{ fromJSON(inputs.build_config).prebuilt_stages }}
      build_runs_on: ${{ fromJSON(inputs.build_config).build_runs_on }}
      rocm_package_version: ${{ inputs.rocm_package_version }}
      release_type: ${{ inputs.release_type }}
      external_repo_config: ${{ inputs.external_repo_config }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  validate_artifact_structure:
    needs: [build_multi_arch_stages]
    name: Validate Artifact Structure
    if: ${{ !failure() && !cancelled() }}
    uses: ./.github/workflows/test_artifacts_structure.yml
    with:
      amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      platform: linux
      release_type: ${{ inputs.release_type }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}

  test_artifacts_per_family:
    needs: [copy_prebuilt_stages, build_multi_arch_stages]
    name: Test ${{ matrix.family_info.amdgpu_family }}
    if: ${{ !failure() && !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
    uses: ./.github/workflows/test_artifacts.yml
    with:
      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
      amdgpu_families: ${{ matrix.family_info.amdgpu_family }}
      amdgpu_targets: ${{ matrix.family_info.amdgpu_targets }}
      test_runs_on: ${{ matrix.family_info.test-runs-on }}
      test_type: ${{ inputs.test_type }}
      # Per-family test_labels override global test_labels when specified
      # Convert array to JSON string for workflow input compatibility
      test_labels: ${{ matrix.family_info.test_labels_for_family && toJSON(matrix.family_info.test_labels_for_family) || inputs.test_labels }}
      sanity_check_only_for_family: ${{ matrix.family_info.sanity_check_only_for_family }}
      changed_projects: ${{ inputs.changed_projects }}
      release_type: ${{ inputs.release_type }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}

  build_native_deb_packages:
    needs: [build_multi_arch_stages]
    name: Build DEB Packages
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml
    with:
      dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      rocm_version: ${{ inputs.rocm_deb_package_version || inputs.rocm_package_version }}
      native_package_type: deb
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  build_native_rpm_packages:
    needs: [build_multi_arch_stages]
    name: Build RPM Packages
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml
    with:
      dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      rocm_version: ${{ inputs.rocm_rpm_package_version || inputs.rocm_package_version }}
      native_package_type: rpm
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  test_native_packages_ubuntu2404:
    needs: [build_native_deb_packages]
    name: Test DEB Install - ubuntu2404
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/test_native_linux_packages_install.yml
    with:
      package_install_url: ${{ needs.build_native_deb_packages.outputs.package_repository_url }}
      release_type: ci
      os_profile: ubuntu2404
      artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      test_type: sanity
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  test_native_packages_rhel8:
    needs: [build_native_rpm_packages]
    name: Test RPM Install - rhel8
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/test_native_linux_packages_install.yml
    with:
      package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }}
      release_type: ci
      os_profile: rhel8
      artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      test_type: sanity
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  test_native_packages_rhel10:
    needs: [build_native_rpm_packages]
    name: Test RPM Install - rhel10
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/test_native_linux_packages_install.yml
    with:
      package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }}
      release_type: ci
      os_profile: rhel10
      artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      test_type: sanity
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  test_native_packages_sles16:
    needs: [build_native_rpm_packages]
    name: Test RPM Install - sles16
    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }}
    uses: ./.github/workflows/test_native_linux_packages_install.yml
    with:
      package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }}
      release_type: ci
      os_profile: sles16
      artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      test_type: sanity
      build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  build_python_packages:
    needs: [build_multi_arch_stages]
    name: Build Python Packages
    if: ${{ !failure() && !cancelled() }}
    uses: ./.github/workflows/build_portable_linux_python_packages.yml
    with:
      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
      amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
      linux_amdgpu_families: ${{ fromJSON(inputs.build_config).linux_amdgpu_families }}
      windows_amdgpu_families: ${{ fromJSON(inputs.build_config).windows_amdgpu_families }}
      multiarch_index: true
      package_version: ${{ inputs.rocm_package_version }}
      release_type: ${{ inputs.release_type }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  test_python_packages_per_family:
    needs: [build_python_packages]
    name: Test Python
    if: ${{ !failure() && !cancelled() && toJSON(fromJSON(inputs.build_config).test_python_packages_matrix) != '[]' }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.build_config).test_python_packages_matrix }}
    uses: ./.github/workflows/test_rocm_wheels.yml
    with:
      amdgpu_family: ${{ matrix.amdgpu_family }}
      test_runs_on: ${{ matrix.test_runs_on }}
      # TODO: Simplify to just `needs.build_python_packages.outputs.package_find_links_url`
      # once kpack split is always enabled.
      package_find_links_url: >-
        ${{
          needs.build_python_packages.outputs.kpack_split == 'true'
          && needs.build_python_packages.outputs.package_find_links_url
          || format('{0}/{1}/index.html',
              needs.build_python_packages.outputs.package_find_links_url,
              matrix.amdgpu_family)
        }}
      python_version: ${{ matrix.python_version }}
      rocm_version: ${{ inputs.rocm_package_version }}
      container_image_name: ${{ matrix.container_image_name }}
      container_image_url: ${{ matrix.container_image_url }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}

  # Multi-arch PyTorch build: one fat wheel covering every target, then
  # split into a host wheel + per-gfx device wheels via rocm-kpack.
  # KPACK_SPLIT_ARTIFACTS is always ON in multi-arch CI (FLAGS.cmake
  # default; build_configure.py's opt-out only fires in single-arch CI),
  # so the split path is unconditional here. The split itself runs inline
  # in multi_arch_build_portable_linux_pytorch_wheels_ci.yml based on
  # inputs.kpack_split.
  build_pytorch_wheel_fat:
    needs: [build_python_packages]
    name: Build PyTorch (fat + split)
    if: >-
      ${{
        !failure() &&
        !cancelled() &&
        fromJSON(inputs.build_config).build_pytorch == true &&
        toJSON(fromJSON(inputs.build_config).pytorch_build_matrix) != '[]'
      }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.build_config).pytorch_build_matrix }}
    uses: ./.github/workflows/multi_arch_build_portable_linux_pytorch_wheels_ci.yml
    with:
      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
      python_version: ${{ matrix.python_version }}
      pytorch_git_ref: ${{ matrix.pytorch_git_ref }}
      # The reusable workflow expands this to gfx targets via
      # cmake/therock_amdgpu_targets.cmake.
      amdgpu_families: ${{ matrix.amdgpu_families }}
      kpack_split: ${{ needs.build_python_packages.outputs.kpack_split }}
      rocm_package_find_links_url: ${{ needs.build_python_packages.outputs.package_find_links_url }}
      rocm_version: ${{ inputs.rocm_package_version }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write

  build_jax_wheels:
    needs: [build_python_packages]
    name: Build JAX Wheels
    if: >-
      ${{
        !failure() &&
        !cancelled() &&
        fromJSON(inputs.build_config).build_jax == true &&
        toJSON(fromJSON(inputs.build_config).jax_build_matrix) != '[]'
      }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.build_config).jax_build_matrix }}
    uses: ./.github/workflows/multi_arch_build_linux_jax_wheels_ci.yml
    with:
      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
      python_version: ${{ matrix.python_version }}
      jax_repository: ${{ matrix.jax_repository }}
      gfx_arch: ${{ matrix.gfx_arch }}
      jax_ref: ${{ matrix.jax_ref }}
      rocm_version: ${{ inputs.rocm_package_version }}
      rocm_package_find_links_url: ${{ needs.build_python_packages.outputs.package_find_links_url }}
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
    permissions:
      contents: read
      id-token: write