diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 958b8354de..0000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,34 +0,0 @@ -# This file itself -/.github/CODEOWNERS @ROCm/TheRock-infra-admins - -# Regularly updated submodules -/rocm-libraries @ROCm/TheRock-infra-write -/rocm-systems @ROCm/TheRock-infra-write - -# New patches are strongly discouraged. See patches/README.md. -/patches/ @ScottTodd @marbre @skganesan008 - -# This file accumulated unrelated code, tighten reviews until cleaned up. -/build_tools/github_actions/github_actions_api.py @ScottTodd - -# Python packages. Changes to these paths must align with -# https://github.com/ROCm/TheRock/blob/main/docs/packaging/python_packaging.md -/build_tools/packaging/python @ScottTodd @marbre -/build_tools/build_python_packages.py @ScottTodd @marbre -/external-builds/pytorch @ScottTodd - -# A few central CI system components that must align with -# https://github.com/ROCm/TheRock/blob/main/docs/development/ci_overview.md -/.github/workflows/setup_multi_arch.yml @ScottTodd -/build_tools/github_actions/configure_multi_arch_ci.py @ScottTodd - -# Debug tools configs and ROCgdb cmake wrapper. -/debug-tools @ROCm/TheRock-infra-write @ROCm/ROCm-DevTools-Debugger -/debug-tools/rocgdb @ROCm/TheRock-infra-write @ROCm/ROCm-DevTools-Debugger - -# Debug tools testing scripts. -/build_tools/github_actions/test_executable_scripts/test_rocgdb.py @ROCm/ROCm-DevTools-Debugger -/build_tools/github_actions/test_executable_scripts/test_rocr-debug-agent.py @ROCm/ROCm-DevTools-Debugger - -# RFCs -/docs/rfcs @ROCm/rocm-product-managers diff --git a/.github/actions/compute_pr_fetch_depth/action.yml b/.github/actions/compute_pr_fetch_depth/action.yml deleted file mode 100644 index 1d697e1495..0000000000 --- a/.github/actions/compute_pr_fetch_depth/action.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: "Compute PR-aware fetch-depth" - -description: >- - Returns the smallest `actions/checkout` fetch-depth that covers all - commits this workflow needs to walk. - - * pull_request events: `commits + 1` (PR commits + merge-base). - * any other event (push, schedule, workflow_dispatch, ...): `0` - (full history). Use this when downstream tooling walks commit - history - -outputs: - value: - description: >- - String value to pass to actions/checkout fetch-depth. Already - coerced to a string so it composes cleanly with checkout's - input parser. - value: ${{ steps.compute.outputs.value }} - -runs: - using: "composite" - steps: - - name: Compute fetch-depth - id: compute - shell: bash - run: python build_tools/github_actions/compute_pr_depth.py diff --git a/.github/actions/configure_aws_artifacts_credentials/action.yml b/.github/actions/configure_aws_artifacts_credentials/action.yml deleted file mode 100644 index 6170250a1d..0000000000 --- a/.github/actions/configure_aws_artifacts_credentials/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: "Configure AWS credentials for artifact uploads" - -description: >- - Determine the IAM role for the artifacts S3 bucket and assume it via OIDC. - * Forks and external repos skip OIDC and use runner base credentials. - * Release builds (dev/nightly/prerelease) use 'therock-{release_type}' roles. - * Other CI runs use the 'therock-ci' role. - See the "Authentication" section in docs/development/s3_buckets.md. - -inputs: - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - default: ci - -runs: - using: "composite" - steps: - - name: Determine IAM role - if: ${{ always() }} - id: iam - shell: bash - run: | - python build_tools/github_actions/write_artifacts_bucket_info.py \ - --release-type "${{ inputs.release_type }}" - - - name: Configure AWS Credentials - if: ${{ always() && steps.iam.outputs.iam_role != '' }} - uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 - with: - aws-region: ${{ steps.iam.outputs.aws_region }} - role-to-assume: ${{ steps.iam.outputs.iam_role }} - special-characters-workaround: ${{ runner.os == 'Windows' }} diff --git a/.github/actions/setup_test_environment/action.yml b/.github/actions/setup_test_environment/action.yml deleted file mode 100644 index db5e992766..0000000000 --- a/.github/actions/setup_test_environment/action.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: "Setup test environment" - -description: Setup ROCm test environment from built artifacts - -inputs: - VENV_DIR: - description: (string) Directory in which to create a Python virtual environment (e.g. `github.workspace/.venv`) - required: true - OUTPUT_ARTIFACTS_DIR: - description: (string) Directory to download/unpack test artifacts into (e.g. `github.workspace/build`) - required: true - ARTIFACT_GROUP: - description: (string) Artifact group (e.g. `gfx110X-all`) - required: true - AMDGPU_TARGETS: - description: (string) Comma-separated individual GPU targets for fetching split artifacts (e.g. `gfx942`) - default: "" - ARTIFACT_RUN_ID: - description: (string) Run ID to download artifacts from (`env.ARTIFACT_RUN_ID` or a value like `19716407405`) - required: true - FETCH_ARTIFACT_ARGS: - description: (string) Extra arguments to install_rocm_from_artifacts.py (e.g. `--blas --tests`) - RELEASE_TYPE: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - default: ci - -runs: - using: "composite" - steps: - - name: "Setting up Python" - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: 3.12 - - - name: "Install UV" - shell: bash - run: | - pip install uv - - - name: Create Python venv - shell: bash - env: - VENV_DIR: ${{ inputs.VENV_DIR }} - run: | - python build_tools/setup_venv.py ${VENV_DIR} \ - --activate-in-future-github-actions-steps \ - --use-uv - - - name: Install test requirements - shell: bash - run: | - uv pip install -r requirements-test.txt - uv pip freeze - - - name: Install Choco tools - if: ${{ runner.os == 'Windows' }} - shell: bash - run: | - choco install --no-progress -y ninja - - - name: Download and Unpack Artifacts - shell: bash - env: - OUTPUT_ARTIFACTS_DIR: ${{ inputs.OUTPUT_ARTIFACTS_DIR }} - ARTIFACT_GROUP: ${{ inputs.ARTIFACT_GROUP }} - AMDGPU_TARGETS: ${{ inputs.AMDGPU_TARGETS }} - ARTIFACT_RUN_ID: ${{ inputs.ARTIFACT_RUN_ID }} - FETCH_ARTIFACT_ARGS: ${{ inputs.FETCH_ARTIFACT_ARGS }} - RELEASE_TYPE: ${{ inputs.RELEASE_TYPE }} - GITHUB_TOKEN: ${{ github.token }} - STEP_NAME: "Download and Unpack Artifacts" - STEP_TIMEOUT_TOLERANCE: "2" - run: | - python ./build_tools/install_rocm_from_artifacts.py \ - --run-id=${ARTIFACT_RUN_ID} \ - --artifact-group=${ARTIFACT_GROUP} \ - --amdgpu-targets=${AMDGPU_TARGETS} \ - --output-dir=${OUTPUT_ARTIFACTS_DIR} \ - ${FETCH_ARTIFACT_ARGS} diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 7996690bf1..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -updates: - - # Check for updates to GitHub Actions - - package-ecosystem: "github-actions" - directories: - - "/" - - "/.github/actions/*" - schedule: - interval: "weekly" - groups: - github-actions: - patterns: - - "*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index b0768fcf9f..0000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,23 +0,0 @@ -## Motivation - - - - - - - -## Technical Details - - - -## Test Plan - - - -## Test Result - - - -## Submission Checklist - -- [ ] Look over the contributing guidelines at https://github.com/ROCm/TheRock/blob/main/CONTRIBUTING.md. diff --git a/.github/workflows/build_portable_linux_python_packages.yml b/.github/workflows/build_portable_linux_python_packages.yml deleted file mode 100644 index e80c5bfc19..0000000000 --- a/.github/workflows/build_portable_linux_python_packages.yml +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Build Portable Linux Python Packages - -on: - workflow_dispatch: - inputs: - artifact_github_repo: - description: "GitHub repository for artifact_run_id" - type: string - default: ROCm/TheRock - artifact_run_id: - description: "artifact_run_id to download artifacts from" - type: string - default: "17865324892" # TODO: default to the most recent successful run (using a script) - artifact_group: - description: "artifact_group to build (e.g. 'gfx94X-dcgpu' for single family builds, 'multi-arch-release' for multi-arch builds with 'amdgpu_families' set)" - type: string - amdgpu_families: - description: "amdgpu_families for multi-arch builds as a semicolon-separated list (e.g. 'gfx94X-dcgpu;gfx120X-all'). Leave empty for single-family builds (those only use artifact_group)" - type: string - default: "" - linux_amdgpu_families: - description: "Linux side of the cross-platform family pair for multi-arch builds; used to advertise the union of device extras in the rocm sdist. Leave empty for single-platform builds." - type: string - default: "" - windows_amdgpu_families: - description: "Windows side of the cross-platform family pair. See linux_amdgpu_families." - type: string - default: "" - multiarch_index: - description: "Enable multi-arch indexing (generates per-family indexes)" - type: boolean - default: false - package_version: - description: "package_version to set on packages, e.g. '7.13.0.dev0+dbd7369489260265fc6ddfc9d0de1262a95fe974'" - type: string - release_type: - description: 'Release type: "ci" for CI, or "dev", "nightly", "prerelease".' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - workflow_call: - inputs: - artifact_github_repo: - type: string - artifact_run_id: - type: string - default: "" - artifact_group: - type: string - amdgpu_families: - type: string - default: "" - linux_amdgpu_families: - type: string - default: "" - windows_amdgpu_families: - type: string - default: "" - multiarch_index: - type: boolean - default: false - package_version: - type: string - release_type: - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - outputs: - package_find_links_url: - description: URL for pip --find-links to install built packages - value: ${{ jobs.build_rocm_wheels.outputs.package_find_links_url }} - kpack_split: - description: "true if this is a kpack-split flat build, false for legacy per-family" - value: ${{ jobs.build_rocm_wheels.outputs.kpack_split }} - -permissions: - contents: read - -run-name: Build portable Linux Python Packages (${{ inputs.artifact_group }}, ${{ inputs.package_version }}) - -jobs: - build_rocm_wheels: - name: Build Python | ${{ inputs.artifact_group }} - # Note: GitHub-hosted runners run out of disk space for some gpu families - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - permissions: - id-token: write - container: - image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - outputs: - package_find_links_url: ${{ steps.upload.outputs.package_find_links_url }} - kpack_split: ${{ steps.upload.outputs.kpack_split }} - env: - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - ARTIFACTS_DIR: "${{ github.workspace }}/artifacts" - PACKAGES_DIR: "${{ github.workspace }}/packages" - RELEASE_TYPE: ${{ inputs.release_type }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Install Python requirements - run: pip install -r requirements.txt - - - name: Fetch artifacts - run: | - if [ -n "${{ inputs.amdgpu_families }}" ]; then - # Multi-arch mode: use artifact_manager.py to fetch all families - echo "Fetching artifacts for multiple families: ${{ inputs.amdgpu_families }}" - python ./build_tools/artifact_manager.py fetch \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --stage=all \ - --amdgpu-families="${{ inputs.amdgpu_families }}" \ - --expand-family-to-targets \ - --output-dir="${{ github.workspace }}" - # NOTE: artifact_manager.py appends /artifacts to --output-dir - # in non-flatten mode, so pass workspace root here so that - # artifacts land in ARTIFACTS_DIR ($workspace/artifacts). - else - # Single-family mode: use fetch_artifacts.py (backward compatible) - echo "Fetching artifacts for single family: ${{ inputs.artifact_group }}" - python ./build_tools/fetch_artifacts.py \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --output-dir="${{ env.ARTIFACTS_DIR }}" - fi - - - name: Build Python packages - run: | - python ./build_tools/build_python_packages.py \ - --artifact-dir="${{ env.ARTIFACTS_DIR }}" \ - --dest-dir="${{ env.PACKAGES_DIR }}" \ - --version="${{ inputs.package_version }}" \ - --linux-amdgpu-families="${{ inputs.linux_amdgpu_families }}" \ - --windows-amdgpu-families="${{ inputs.windows_amdgpu_families }}" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - # NOTE: we use `github.run_id` and NOT `env.ARTIFACT_RUN_ID` here! - # This ensures that if they are different we _download_ artifacts from the - # input run's subdirectory and _upload_ to our current run's subdirectory. - - name: Upload Python packages - id: upload - run: | - python build_tools/github_actions/upload_python_packages.py \ - --input-packages-dir="${{ env.PACKAGES_DIR }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --run-id="${{ github.run_id }}" \ - ${{ inputs.multiarch_index && '--multiarch' || '' }} diff --git a/.github/workflows/build_windows_python_packages.yml b/.github/workflows/build_windows_python_packages.yml deleted file mode 100644 index fc8b039bc8..0000000000 --- a/.github/workflows/build_windows_python_packages.yml +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Build Windows Python Packages - -on: - workflow_dispatch: - inputs: - artifact_github_repo: - description: "GitHub repository for artifact_run_id" - type: string - default: ROCm/TheRock - artifact_run_id: - description: "artifact_run_id to download artifacts from" - type: string - default: "17865324892" # TODO: default to the most recent successful run (using a script) - artifact_group: - description: "artifact_group to build (e.g. 'gfx94X-dcgpu' for single family builds, 'multi-arch-release' for multi-arch builds with 'amdgpu_families' set)" - type: string - amdgpu_families: - description: "amdgpu_families for multi-arch builds as a semicolon-separated list (e.g. 'gfx94X-dcgpu;gfx120X-all'). Leave empty for single-family builds (those only use artifact_group)" - type: string - default: "" - linux_amdgpu_families: - description: "Linux side of the cross-platform family pair for multi-arch builds; used to advertise the union of device extras in the rocm sdist. Leave empty for single-platform builds." - type: string - default: "" - windows_amdgpu_families: - description: "Windows side of the cross-platform family pair. See linux_amdgpu_families." - type: string - default: "" - multiarch_index: - description: "Enable multi-arch indexing (generates per-family indexes)" - type: boolean - default: false - package_version: - description: "package_version to set on packages, e.g. '7.13.0.dev0+dbd7369489260265fc6ddfc9d0de1262a95fe974'" - type: string - release_type: - description: 'Release type: "ci" for CI, or "dev", "nightly", "prerelease".' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - workflow_call: - inputs: - artifact_github_repo: - type: string - artifact_run_id: - type: string - default: "" - artifact_group: - type: string - amdgpu_families: - type: string - default: "" - linux_amdgpu_families: - type: string - default: "" - windows_amdgpu_families: - type: string - default: "" - multiarch_index: - type: boolean - default: false - package_version: - type: string - release_type: - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - outputs: - package_find_links_url: - description: URL for pip --find-links to install built packages - value: ${{ jobs.build_rocm_wheels.outputs.package_find_links_url }} - kpack_split: - description: "true if this is a kpack-split flat build, false for legacy per-family" - value: ${{ jobs.build_rocm_wheels.outputs.kpack_split }} - -permissions: - contents: read - -run-name: Build Windows Python Packages (${{ inputs.artifact_group }}, ${{ inputs.package_version }}) - -jobs: - build_rocm_wheels: - name: Build Python | ${{ inputs.artifact_group }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-windows-scale-rocm' || 'windows-2022' }} - permissions: - id-token: write - outputs: - package_find_links_url: ${{ steps.upload.outputs.package_find_links_url }} - kpack_split: ${{ steps.upload.outputs.kpack_split }} - env: - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - ARTIFACTS_DIR: "${{ github.workspace }}/artifacts" - PACKAGES_DIR: "${{ github.workspace }}/packages" - RELEASE_TYPE: ${{ inputs.release_type }} - defaults: - run: - shell: bash - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - - name: Install Python requirements - run: pip install -r requirements.txt - - - name: Fetch artifacts - run: | - if [ -n "${{ inputs.amdgpu_families }}" ]; then - # Multi-arch mode: use artifact_manager.py to fetch all families - echo "Fetching artifacts for multiple families: ${{ inputs.amdgpu_families }}" - python ./build_tools/artifact_manager.py fetch \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --stage=all \ - --amdgpu-families="${{ inputs.amdgpu_families }}" \ - --expand-family-to-targets \ - --output-dir="${{ github.workspace }}" - # NOTE: artifact_manager.py appends /artifacts to --output-dir - # in non-flatten mode, so pass workspace root here so that - # artifacts land in ARTIFACTS_DIR ($workspace/artifacts). - else - # Single-family mode: use fetch_artifacts.py (backward compatible) - echo "Fetching artifacts for single family: ${{ inputs.artifact_group }}" - python ./build_tools/fetch_artifacts.py \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --output-dir="${{ env.ARTIFACTS_DIR }}" - fi - - - name: Build Python packages - run: | - python ./build_tools/build_python_packages.py \ - --artifact-dir="${{ env.ARTIFACTS_DIR }}" \ - --dest-dir="${{ env.PACKAGES_DIR }}" \ - --version="${{ inputs.package_version }}" \ - --linux-amdgpu-families="${{ inputs.linux_amdgpu_families }}" \ - --windows-amdgpu-families="${{ inputs.windows_amdgpu_families }}" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - # NOTE: we use `github.run_id` and NOT `env.ARTIFACT_RUN_ID` here! - # This ensures that if they are different we _download_ artifacts from the - # input run's subdirectory and _upload_ to our current run's subdirectory. - - name: Upload Python packages - id: upload - run: | - python build_tools/github_actions/upload_python_packages.py \ - --input-packages-dir="${{ env.PACKAGES_DIR }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --run-id="${{ github.run_id }}" \ - ${{ inputs.multiarch_index && '--multiarch' || '' }} diff --git a/.github/workflows/bump_submodules.yml b/.github/workflows/bump_submodules.yml deleted file mode 100644 index 3fa9be5bd1..0000000000 --- a/.github/workflows/bump_submodules.yml +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Bump Submodule automation - -on: - workflow_dispatch: - inputs: - submodule: - description: 'Submodule to bump (default: all)' - required: false - default: 'all' - type: choice - options: - - all - - rocm-systems - - rocm-libraries - - rocgdb - schedule: - - cron: "0 */12 * * *" # Runs every 12 hrs at 12AM and 12PM - push: - branches: - - main - paths: - - rocm-systems - - rocm-libraries - - debug-tools/rocgdb/source - -permissions: - contents: read - -jobs: - bump-submodules: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout ROCm/TheRock - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.11' - - - name: Generate GitHub App token rocm-systems - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - id: systems-token - with: - app-id: ${{ secrets.ROCM_SYSTEMS_APP_ID }} - private-key: ${{ secrets.ROCM_SYSTEMS_APP_PRIVATE_KEY }} - owner: "ROCm" - # We will reuse the rocm-systems token for rocgdb for now. - repositories: "rocm-systems,rocgdb,TheRock" - - - name: Generate GitHub App token rocm-libraries - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - id: librarian-token - with: - app-id: ${{ secrets.ROCM_LIBRARIES_APP_ID }} - private-key: ${{ secrets.ROCM_LIBRARIES_APP_PRIVATE_KEY }} - owner: "ROCm" - repositories: "rocm-libraries,TheRock" - - - name: Install prerequisites - run: | - pip install requests - - - name: Run scheduled bump automation - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - run: | - python3 build_tools/github_actions/bump_automation.py \ - --event_type "schedule" \ - --submodule "${{ github.event.inputs.submodule || 'all' }}" \ - --systems_token "${{ steps.systems-token.outputs.token }}" \ - --libraries_token "${{ steps.librarian-token.outputs.token }}" \ - --rocgdb_token "${{ steps.systems-token.outputs.token }}" - - - name: Run push automation - if: github.event_name == 'push' - run: | - python3 build_tools/github_actions/bump_automation.py \ - --event_type "push" \ - --before "${{ github.event.before }}" \ - --after "${{ github.sha }}" \ - --systems_token "${{ steps.systems-token.outputs.token }}" \ - --libraries_token "${{ steps.librarian-token.outputs.token }}" \ - --rocgdb_token "${{ steps.systems-token.outputs.token }}" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index e9461b035a..0000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -name: CodeQL - -on: - push: - branches: [main] - paths: - - '**/*.py' - - '.github/**' - pull_request: - branches: [main] - paths: - - '**/*.py' - - '.github/**' - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - analyze: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false - name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-24.04 - timeout-minutes: 30 - permissions: - contents: read - security-events: write - strategy: - fail-fast: false - matrix: - language: [python, actions] - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Initialize CodeQL - uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 - with: - languages: ${{ matrix.language }} - queries: security-extended - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 - with: - category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/copy_release.yml b/.github/workflows/copy_release.yml deleted file mode 100644 index 1af5bfa940..0000000000 --- a/.github/workflows/copy_release.yml +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Copy release to dev bucket - -on: - workflow_dispatch: - inputs: - rocm_version: - description: ROCm version to copy from nightly releases to dev releases, e.g. 7.14.0a20260617 (see https://rocm.nightlies.amd.com/whl-multi-arch/) - type: string - python_version: - type: choice - options: - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - default: "3.12" - include_torch: - description: Copy 'torch' packages in addition to 'rocm' packages? - type: boolean - default: false - include_jax: - description: Copy 'jax' packages in addition to 'rocm' packages? - type: boolean - default: false -permissions: - contents: read - -run-name: >- - Copy nightly -> dev | - rocm ${{ inputs.rocm_version }} | - py ${{ inputs.python_version }} | - extras: ${{ - inputs.include_torch && inputs.include_jax && 'PyTorch + JAX' || - inputs.include_torch && 'PyTorch' || - inputs.include_jax && 'JAX' || - 'none' - }} - -jobs: - copy_python_packages: - name: Copy nightly -> dev | rocm ${{ inputs.rocm_version }} | py ${{ inputs.python_version }} - runs-on: ubuntu-24.04 - permissions: - id-token: write - - steps: - - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # TODO: migrate to boto3 Python package instead of this native tool? - - name: Install the AWS tool - run: ./dockerfiles/install_awscli.sh - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-dev - - - name: Select Python version - run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} - - - name: Copy ROCm packages between S3 buckets - run: | - aws s3 cp \ - s3://therock-nightly-python/v4/whl/ \ - s3://therock-dev-python/v4/whl/ \ - --recursive --exclude "*" \ - --include "rocm*${{ inputs.rocm_version }}*" - - - name: Copy torch wheels between S3 buckets - if: ${{ inputs.include_torch }} - run: | - aws s3 cp \ - s3://therock-nightly-python/v4/whl/ \ - s3://therock-dev-python/v4/whl/ \ - --recursive --exclude "*" \ - --include "*torch*${{ inputs.rocm_version }}*${{ env.cp_version }}*" \ - --include "apex*${{ inputs.rocm_version }}*${{ env.cp_version }}*" \ - --include "triton*${{ inputs.rocm_version }}*${{ env.cp_version }}*" - - - name: Copy JAX wheels between S3 buckets - if: ${{ inputs.include_jax }} - run: | - aws s3 cp \ - s3://therock-nightly-python/v4/whl/ \ - s3://therock-dev-python/v4/whl/ \ - --recursive --exclude "*" \ - --include "jax*${{ inputs.rocm_version }}*${{ env.cp_version }}*" diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml deleted file mode 100644 index 356deb379b..0000000000 --- a/.github/workflows/gitleaks.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -name: Gitleaks - -on: - workflow_call: - inputs: - scan_mode: - description: >- - 'changed' (default) scans only commits introduced by the calling - event; requires a pull_request or push - event payload and hard-fails otherwise. 'all' scans the full - repository history and is required for schedule, - workflow_dispatch, release, and any other event type. - required: false - type: string - default: changed - report_formats: - description: >- - Comma-separated list of gitleaks report formats to produce. Allowed - values: 'sarif' (default), 'json', 'csv', 'junit'. SARIF outputs are - uploaded to GitHub code scanning via github/codeql-action/upload-sarif; - all other formats are uploaded as build artifacts via - actions/upload-artifact. Example: 'sarif,json'. - required: false - type: string - default: sarif - scan_path: - description: >- - Path (relative to the repository root) to scan. Defaults to '.' so - the entire superproject checkout is scanned; set to a subdirectory - (e.g. 'src/' or 'services/api/') to restrict the scan to that - subtree. The path MUST exist after checkout or the scan job fails. - Submodules are NOT scanned: the checkout step leaves them - uninitialized (its default), and each submodule is a separately - versioned ROCm or third-party repo with its own gitleaks coverage. - A PR that bumps a submodule SHA surfaces as a one-line change in - the superproject diff and is reviewable on its own. - required: false - type: string - default: "." -# Reusable workflows declare the *minimum* permissions every caller -# must grant. -permissions: - contents: read - -jobs: - scan: - name: Gitleaks scan - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Bootstrap checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 1 - - - name: Compute checkout fetch-depth - id: depth - uses: ./.github/actions/compute_pr_fetch_depth - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: ${{ steps.depth.outputs.value }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Run gitleaks - id: scan - env: - GITLEAKS_SCAN_MODE: ${{ inputs.scan_mode }} - GITLEAKS_REPORT_FORMATS: ${{ inputs.report_formats }} - GITLEAKS_SOURCE_DIR: ${{ inputs.scan_path }} - run: python build_tools/scan_tools/github_actions/gitleaks.py - - - name: Upload SARIF report to code scanning - if: always() && steps.scan.outputs.sarif_path != '' - uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 - with: - sarif_file: ${{ steps.scan.outputs.sarif_path }} - category: gitleaks - - - name: Upload non-SARIF reports - if: always() && steps.scan.outputs.non_sarif_paths != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: gitleaks-report - path: ${{ steps.scan.outputs.non_sarif_paths }} - if-no-files-found: ignore diff --git a/.github/workflows/gitleaks_main.yml b/.github/workflows/gitleaks_main.yml deleted file mode 100644 index 45e21490d7..0000000000 --- a/.github/workflows/gitleaks_main.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -# Weekly gitleaks scan: runs on a fixed cadence and pushes -# SARIF findings to the repository's code-scanning Security tab. -name: Gitleaks main - -on: - # Run every Saturday at 10:00 UTC. - schedule: - - cron: "0 10 * * 6" - # Allow re-running the weekly scan on demand, e.g. after - # tweaking `gitleaks.toml` or rotating the gitleaks version pin, so - # the Security tab can be refreshed without an unrelated commit. - workflow_dispatch: - -permissions: - contents: read - -jobs: - gitleaks: - uses: ./.github/workflows/gitleaks.yml - with: - scan_mode: all - report_formats: sarif - permissions: - contents: read - # Required so the reusable workflow can call - # github/codeql-action/upload-sarif and have the findings - # appear under Security -> Code scanning, filterable by - # `Tool: gitleaks` separately from any other scanner. - security-events: write diff --git a/.github/workflows/hip_tagging_automation.yml b/.github/workflows/hip_tagging_automation.yml deleted file mode 100644 index 1abba1afbf..0000000000 --- a/.github/workflows/hip_tagging_automation.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Hip tagging automation - -on: - push: - branches: - - main - paths: - - rocm-systems - -permissions: - contents: read - -jobs: - tag-rocm-systems: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout ROCm/TheRock - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.11' - - - name: Generate GitHub App token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - id: generate-token - with: - app-id: ${{ secrets.ROCM_SYSTEMS_APP_ID }} - private-key: ${{ secrets.ROCM_SYSTEMS_APP_PRIVATE_KEY }} - owner: "ROCm" - repositories: "rocm-systems" - - - name: Install dependencies - run: | - pip install PyGithub - - - name: Run tagging script - run: | - python build_tools/github_actions/hip_tagging_helper.py \ - --repo "${{ github.repository }}" \ - --before "${{ github.event.before }}" \ - --after "${{ github.sha }}" \ - --pat "${{ steps.generate-token.outputs.token }}" diff --git a/.github/workflows/manifest-diff.yml b/.github/workflows/manifest-diff.yml deleted file mode 100644 index 3cbc91afb7..0000000000 --- a/.github/workflows/manifest-diff.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: TheRock Manifest Diff Report - -on: - workflow_dispatch: - inputs: - start_ref: - description: 'Start commit SHA or workflow run ID. Used when neither pr_base_ref nor find_last_run is set (start resolution precedence: pr_base_ref > find_last_run > start_ref).' - required: false - type: string - end_ref: - description: 'End commit SHA (or, with workflow_mode=true, a workflow run ID).' - required: true - type: string - workflow_mode: - description: 'Interpret start_ref and end_ref as workflow run IDs instead of commit SHAs (pr_base_ref and find_last_run are unaffected).' - required: false - type: boolean - default: false - find_last_run: - description: 'Workflow filename (e.g. multi_arch_ci.yml). When set, start_ref is resolved as the head SHA of that workflow''s most recent run on `branch` that concluded with success or failure (cancelled / skipped / in-progress runs are ignored).' - required: false - type: string - pr_base_ref: - description: 'PR base branch name (e.g. main). When set, start_ref is resolved as the merge-base between end_ref and the named branch via the GitHub Compare API — rebase-safe, works on rewritten PR histories.' - required: false - type: string - branch: - description: 'Branch to scope find_last_run lookups against (default: main). Only consulted when find_last_run is set; ignored otherwise.' - required: false - type: string - default: 'main' - repository: - description: 'Repository to check out (default: the repository hosting this workflow run).' - required: false - type: string - default: '' - ref: - description: 'Git ref to check out (default: the ref hosting this workflow run).' - required: false - type: string - default: '' - workflow_call: - inputs: - start_ref: - required: false - type: string - end_ref: - required: false - type: string - default: '' - workflow_mode: - required: false - type: boolean - default: false - find_last_run: - required: false - type: string - pr_base_ref: - required: false - type: string - branch: - required: false - type: string - default: 'main' - repository: - required: false - type: string - default: '' - ref: - required: false - type: string - default: '' - -permissions: - contents: read - -jobs: - generate-report: - name: Generate Manifest Diff Report - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - # Non-blocking under CI; strict on manual dispatch so regressions surface red. - continue-on-error: ${{ github.event_name != 'workflow_dispatch' }} - permissions: - id-token: write - contents: read - container: - image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - fetch-depth: 0 - - - name: Adjust git config - run: | - git config --global --add safe.directory $PWD - - - name: Generate manifest diff report - id: gen-report - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python3 build_tools/generate_manifest_diff_report.py \ - --end "${{ inputs.end_ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" \ - --start "${{ inputs.start_ref || (github.event_name == 'push' && github.event.before) || '' }}" \ - --find-last-run "${{ inputs.find_last_run }}" \ - --pr-base-ref "${{ inputs.pr_base_ref || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || '' }}" \ - --branch "${{ inputs.branch }}" \ - ${{ inputs.workflow_mode && '--workflow-mode' || '' }} \ - --output-dir reports - - - name: Configure AWS credentials for artifact uploads - if: ${{ steps.gen-report.conclusion == 'success' }} - uses: ./.github/actions/configure_aws_artifacts_credentials - - - name: Upload Report to S3 - if: ${{ steps.gen-report.conclusion == 'success' }} - run: | - python3 build_tools/github_actions/upload_test_report_script.py \ - --run-id ${{ github.run_id }} \ - --subfolder manifest-diff \ - --report-path reports \ - --index-file-name index.html diff --git a/.github/workflows/multi_arch_build_linux_jax_wheels.yml b/.github/workflows/multi_arch_build_linux_jax_wheels.yml deleted file mode 100644 index 3d17aa9d62..0000000000 --- a/.github/workflows/multi_arch_build_linux_jax_wheels.yml +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable single-build workflow for the Linux JAX wheel flow. -# -# This workflow is called once per release matrix cell from -# test_linux_jax_wheels_partial.yml -# -# Built shape: -# - One build/ci_build invocation for one python_version and jax_ref -# matrix cell. -# - Uploads the resulting wheels to the multi-arch release bucket. -# - Runs the JAX test subset against the published multi-arch index. - -name: Multi-Arch Build Linux JAX Wheels - -on: - workflow_call: - inputs: - test_amdgpu_family: - description: "GPU family to use for JAX tests; empty means skip tests." - type: string - default: "" - python_version: - description: "Python version to build wheels for." - type: string - required: true - jax_repository: - description: "Repository containing the JAX release branch." - type: string - required: true - gfx_arch: - description: "ROCm device package selector used for the manylinux build (e.g. device-gfx942, device-gfx950, or device-all)." - type: string - required: false - default: "" - jax_ref: - description: "JAX / rocm-jax release ref." - type: string - required: true - rocm_version: - description: "ROCm package version to install and build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - release_type: - description: "Release type." - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - outputs: - package_index_url: - description: "Package index URL for the multi-arch release bucket." - value: ${{ jobs.build_jax_wheels.outputs.package_index_url }} - jax_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_version }} - jaxlib_version: - value: ${{ jobs.build_jax_wheels.outputs.jaxlib_version }} - jax_plugin_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_plugin_version }} - jax_pjrt_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_pjrt_version }} - workflow_dispatch: - inputs: - test_amdgpu_family: - description: "Representative GPU family to use for JAX tests." - type: string - default: "gfx94X-dcgpu" - python_version: - description: "Python version to build wheels for." - type: string - required: true - jax_repository: - description: "GitHub repository in owner/repo format (e.g. ROCm/jax or ROCm/rocm-jax)." - type: string - default: "ROCm/jax" - gfx_arch: - description: "ROCm device package selector used for the manylinux build (e.g. device-gfx942, device-gfx950, or device-all)." - type: string - required: false - default: "" - jax_ref: - description: "JAX / rocm-jax release ref." - type: string - required: true - rocm_version: - description: "ROCm package version to install and build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - release_type: - description: "Release type." - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" -run-name: Build Multi-Arch Linux JAX Wheels (${{ inputs.release_type }}, ${{ inputs.rocm_version }}, Multiarch) - -permissions: - contents: read - id-token: write - -jobs: - build_jax_wheels: - name: Build | py ${{ inputs.python_version }} | jax ${{ inputs.jax_ref }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-linux-scale-rocm' || 'ubuntu-24.04' }} - outputs: - package_index_url: ${{ steps.publish_jax_wheels.outputs.package_index_url }} - jax_version: ${{ steps.write_jax_versions.outputs.jax_version }} - jaxlib_version: ${{ steps.write_jax_versions.outputs.jaxlib_version }} - jax_plugin_version: ${{ steps.write_jax_versions.outputs.jax_plugin_version }} - jax_pjrt_version: ${{ steps.write_jax_versions.outputs.jax_pjrt_version }} - env: - MANYLINUX_IMAGE_TAG: therock-jax-manylinux:${{ inputs.jax_ref }}-${{ inputs.python_version }} - PACKAGE_DIST_DIR: ${{ github.workspace }}/jax-source/dist - - steps: - - name: Checkout TheRock - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || github.ref_name }} - - - name: Checkout rocm-jax - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: rocm-jax - repository: ROCm/rocm-jax - ref: ${{ inputs.jax_ref }} - - - name: Checkout JAX - id: checkout_jax - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: jax-source - repository: ${{ inputs.jax_repository }} - ref: ${{ inputs.jax_ref }} - - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - - name: Install python deps for CI - run: | - pip install -r external-builds/jax/requirements-jax.txt - - # TODO: THEROCK_INDEX_URL need to be renamed in the future to - # THEROCK_FIND_LINKS_URL to avoid confusion with pip --index-url. - - name: Build manylinux image - working-directory: jax-source - env: - THEROCK_VERSION: "" - GFX_ARCH: ${{ inputs.gfx_arch }} - ROCM_PACKAGE_FIND_LINKS_URL: ${{ inputs.rocm_package_find_links_url }} - run: | - cp ../rocm-jax/docker/manylinux/Dockerfile.jax-manylinux_2_28-therock \ - Dockerfile.jax-manylinux_2_28-therock - - docker build \ - -t "${MANYLINUX_IMAGE_TAG}" \ - --file=Dockerfile.jax-manylinux_2_28-therock \ - --build-arg=THEROCK_INDEX_URL="${ROCM_PACKAGE_FIND_LINKS_URL}" \ - --build-arg=THEROCK_VERSION="${THEROCK_VERSION}" \ - --build-arg=GFX_ARCH="${GFX_ARCH}" \ - --progress=plain \ - . - - - name: Determine wheel version suffix - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version "${{ inputs.rocm_version }}" - - - name: Build JAX Wheels in manylinux container - working-directory: jax-source - env: - ROCM_VERSION: ${{ inputs.rocm_version }} - PYTHON_VERSION: ${{ inputs.python_version }} - run: | - docker run --rm \ - --user root \ - --env ROCM_VERSION="${ROCM_VERSION}" \ - --env PYTHON_VERSION="${PYTHON_VERSION}" \ - --env ML_WHEEL_VERSION_SUFFIX="${version_suffix}" \ - --env PACKAGE_DIST_DIR="/workspace/jax-source/dist" \ - --volume "${GITHUB_WORKSPACE}:/workspace" \ - --workdir /workspace/jax-source \ - "${MANYLINUX_IMAGE_TAG}" \ - bash -lc ' - python build/build.py build \ - --wheels=jax-rocm-plugin,jax-rocm-pjrt \ - --python_version="${PYTHON_VERSION}" \ - --bazel_startup_options=--bazelrc=build/rocm/rocm.bazelrc \ - --bazel_options=--config=rocm_release_wheel \ - --bazel_options=--repo_env=ROCM_PATH=$(rocm-sdk path --root) \ - --bazel_options=--repo_env=ML_WHEEL_TYPE=release \ - --bazel_options=--repo_env=ML_WHEEL_VERSION_SUFFIX="${ML_WHEEL_VERSION_SUFFIX}" \ - --bazel_options=--//jaxlib/tools:jaxlib_git_hash="${{ steps.checkout_jax.outputs.commit }}" \ - --verbose \ - --detailed_timestamped_log \ - --output_path=$(pwd)/dist - ' - - - name: Extract JAX versions from built wheels - id: write_jax_versions - run: | - python3 ./build_tools/github_actions/write_jax_versions.py \ - --dist-dir "${{ env.PACKAGE_DIST_DIR }}" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Upload JAX wheels to release bucket - id: publish_jax_wheels - run: | - python build_tools/github_actions/publish_jax_to_release_bucket.py \ - --source-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --release-type="${{ inputs.release_type }}" - - generate_target_to_run: - name: Generate target_to_run - if: ${{ inputs.test_amdgpu_family != '' }} - runs-on: ubuntu-24.04 - outputs: - test_runs_on: ${{ steps.configure.outputs.test-runs-on }} - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || github.ref_name }} - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - - name: Generating target to run - id: configure - env: - TARGET: ${{ inputs.test_amdgpu_family }} - PLATFORM: linux - CI_CONFIG_PATH: ci-config - run: python ./build_tools/github_actions/configure_target_run.py - - test_jax_wheels: - if: ${{ inputs.test_amdgpu_family != '' }} - needs: [build_jax_wheels, generate_target_to_run] - name: Test JAX wheels | py ${{ inputs.python_version }} | jax ${{ inputs.jax_ref }} - permissions: - contents: read - id-token: write - uses: ./.github/workflows/test_linux_jax_wheels_partial.yml - with: - test_amdgpu_family: ${{ inputs.test_amdgpu_family }} - package_index_url: ${{ needs.build_jax_wheels.outputs.package_index_url }} - rocm_version: ${{ inputs.rocm_version }} - rocm_package_find_links_url: ${{ inputs.rocm_package_find_links_url }} - release_type: ${{ inputs.release_type }} - python_version: ${{ inputs.python_version }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - jax_ref: ${{ inputs.jax_ref }} - jax_version: ${{ needs.build_jax_wheels.outputs.jax_version }} - jaxlib_version: ${{ needs.build_jax_wheels.outputs.jaxlib_version }} - jax_plugin_version: ${{ needs.build_jax_wheels.outputs.jax_plugin_version }} - jax_pjrt_version: ${{ needs.build_jax_wheels.outputs.jax_pjrt_version }} - test_runs_on: ${{ needs.generate_target_to_run.outputs.test_runs_on }} diff --git a/.github/workflows/multi_arch_build_linux_jax_wheels_ci.yml b/.github/workflows/multi_arch_build_linux_jax_wheels_ci.yml deleted file mode 100644 index 3592c6bfa0..0000000000 --- a/.github/workflows/multi_arch_build_linux_jax_wheels_ci.yml +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable single-build workflow for the Linux JAX wheel CI flow. -# -# This workflow is called once per CI matrix cell from multi_arch_ci_linux.yml. -# -# Built shape: -# - One manylinux JAX wheel build for one python_version and jax_ref matrix cell. -# - Installs ROCm packages from the run-scoped CI find-links URL. -# - Uploads the resulting wheels to run-scoped CI Python artifacts. -# - Build-only for the first CI rollout; test wiring should be added separately. - -name: Multi-Arch Build Linux JAX Wheels (CI) - -on: - workflow_call: - inputs: - artifact_group: - description: "Artifact group for CI artifact upload." - type: string - required: true - python_version: - type: string - default: "3.12" - jax_repository: - description: "Repository containing the JAX release branch." - type: string - required: true - gfx_arch: - description: "ROCm device package selector used for the manylinux build (e.g. device-gfx942, device-gfx950, or device-all)." - type: string - required: false - default: "" - jax_ref: - description: "JAX / rocm-jax release ref." - type: string - required: true - rocm_version: - description: "ROCm package version to install and build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - outputs: - package_find_links_url: - description: "pip --find-links URL for CI-uploaded JAX wheels." - value: ${{ jobs.build_jax_wheels.outputs.package_find_links_url }} - jax_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_version }} - jaxlib_version: - value: ${{ jobs.build_jax_wheels.outputs.jaxlib_version }} - jax_plugin_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_plugin_version }} - jax_pjrt_version: - value: ${{ jobs.build_jax_wheels.outputs.jax_pjrt_version }} - workflow_dispatch: - inputs: - artifact_group: - description: "Artifact group for CI artifact upload." - type: string - required: true - python_version: - type: string - default: "3.12" - jax_repository: - description: "GitHub repository in owner/repo format containing the JAX release branch." - type: string - default: "ROCm/jax" - gfx_arch: - description: "ROCm device package selector used for the manylinux build (e.g. device-gfx942, device-gfx950, or device-all)." - type: string - required: false - default: "" - jax_ref: - description: "JAX / rocm-jax release ref (e.g. rocm-jaxlib-v0.10.0, rocm-jaxlib-v0.10.2)." - type: string - required: true - rocm_version: - description: "ROCm package version to install and build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" -run-name: Build Multi-Arch Linux JAX Wheels CI (${{ inputs.artifact_group }}, ${{ inputs.rocm_version }}, py${{ inputs.python_version }}, ${{ inputs.jax_ref }}) -permissions: - contents: read - id-token: write - -jobs: - build_jax_wheels: - name: Build JAX | ${{ inputs.artifact_group }} | jax ${{ inputs.jax_ref }} | py ${{ inputs.python_version }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-linux-scale-rocm' || 'ubuntu-24.04' }} - outputs: - package_find_links_url: ${{ steps.upload_jax_wheels.outputs.package_find_links_url }} - jax_version: ${{ steps.write_jax_versions.outputs.jax_version }} - jaxlib_version: ${{ steps.write_jax_versions.outputs.jaxlib_version }} - jax_plugin_version: ${{ steps.write_jax_versions.outputs.jax_plugin_version }} - jax_pjrt_version: ${{ steps.write_jax_versions.outputs.jax_pjrt_version }} - env: - MANYLINUX_IMAGE_TAG: therock-jax-manylinux:${{ inputs.jax_ref }}-${{ inputs.python_version }} - RELEASE_TYPE: ci - PACKAGE_DIST_DIR: ${{ github.workspace }}/jax-source/dist - - steps: - - name: Checkout TheRock - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout rocm-jax - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: rocm-jax - repository: ROCm/rocm-jax - ref: ${{ inputs.jax_ref }} - - - name: Checkout JAX - id: checkout_jax - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: jax-source - repository: ${{ inputs.jax_repository }} - ref: ${{ inputs.jax_ref }} - - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - - name: Install python deps for CI - run: | - pip install -r external-builds/jax/requirements-jax.txt - - - name: Build manylinux image - working-directory: jax-source - env: - THEROCK_VERSION: "" - GFX_ARCH: ${{ inputs.gfx_arch }} - ROCM_PACKAGE_FIND_LINKS_URL: ${{ inputs.rocm_package_find_links_url }} - run: | - cp ../rocm-jax/docker/manylinux/Dockerfile.jax-manylinux_2_28-therock \ - Dockerfile.jax-manylinux_2_28-therock - - # TODO(#6388): Replace this per-run rocm-jax Dockerfile build with a - # frozen/prebuilt CI image and refine when JAX CI should run. - docker build \ - -t "${MANYLINUX_IMAGE_TAG}" \ - --file=Dockerfile.jax-manylinux_2_28-therock \ - --build-arg=THEROCK_INDEX_URL="${ROCM_PACKAGE_FIND_LINKS_URL}" \ - --build-arg=THEROCK_VERSION="${THEROCK_VERSION}" \ - --build-arg=GFX_ARCH="${GFX_ARCH}" \ - --progress=plain \ - . - - - name: Determine wheel version suffix - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version "${{ inputs.rocm_version }}" - - - name: Build JAX Wheels in manylinux container - working-directory: jax-source - env: - ROCM_VERSION: ${{ inputs.rocm_version }} - PYTHON_VERSION: ${{ inputs.python_version }} - run: | - docker run --rm \ - --user root \ - --env ROCM_VERSION="${ROCM_VERSION}" \ - --env PYTHON_VERSION="${PYTHON_VERSION}" \ - --env ML_WHEEL_VERSION_SUFFIX="${version_suffix}" \ - --env PACKAGE_DIST_DIR="/workspace/jax-source/dist" \ - --volume "${GITHUB_WORKSPACE}:/workspace" \ - --workdir /workspace/jax-source \ - "${MANYLINUX_IMAGE_TAG}" \ - bash -lc ' - python build/build.py build \ - --wheels=jax-rocm-plugin,jax-rocm-pjrt \ - --python_version="${PYTHON_VERSION}" \ - --bazel_startup_options=--bazelrc=build/rocm/rocm.bazelrc \ - --bazel_options=--config=rocm_release_wheel \ - --bazel_options=--repo_env=ROCM_PATH=$(rocm-sdk path --root) \ - --bazel_options=--repo_env=ML_WHEEL_TYPE=release \ - --bazel_options=--repo_env=ML_WHEEL_VERSION_SUFFIX="${ML_WHEEL_VERSION_SUFFIX}" \ - --bazel_options=--//jaxlib/tools:jaxlib_git_hash="${{ steps.checkout_jax.outputs.commit }}" \ - --verbose \ - --detailed_timestamped_log \ - --output_path=$(pwd)/dist - ' - - - name: Extract JAX versions from built wheels - id: write_jax_versions - run: | - python3 ./build_tools/github_actions/write_jax_versions.py \ - --dist-dir "${{ env.PACKAGE_DIST_DIR }}" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ci - - - name: Upload JAX wheels to CI artifacts - id: upload_jax_wheels - run: | - python build_tools/github_actions/upload_python_packages.py \ - --input-packages-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --run-id="${{ github.run_id }}" \ - --multiarch diff --git a/.github/workflows/multi_arch_build_native_linux_packages.yml b/.github/workflows/multi_arch_build_native_linux_packages.yml deleted file mode 100644 index e8857a26e4..0000000000 --- a/.github/workflows/multi_arch_build_native_linux_packages.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Build Multi-Arch Native Linux Packages - -on: - workflow_call: - inputs: - dist_amdgpu_families: - description: "Semicolon-separated list of all GPU families (e.g., 'gfx94X-dcgpu;gfx120X-all')" - required: true - type: string - artifact_run_id: - description: "Workflow run id to download the artifacts from" - required: false - type: string - default: "" - rocm_version: - description: "ROCm version to append to the package (8.0.0, 8.0.1rc1, ...)" - required: true - type: string - native_package_type: - description: "Package type (deb or rpm)" - required: true - type: string - package_suffix: - description: "The suffix to be added to package name (asan, tsan, static or rpath)" - required: false - type: string - default: "" - build_variant: - description: "Build variant (e.g. 'asan'). When set to 'asan', changes the install prefix to DEFAULT_INSTALL_PREFIX-asan-MAJOR.MINOR (e.g. /opt/rocm/core-asan-7.15)." - required: false - type: string - default: "" - release_type: - description: "Release type: 'ci', 'dev', 'nightly', or 'prerelease'" - required: false - type: string - default: ci - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`" - type: string - required: false - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow" - type: string - required: false - outputs: - package_repository_url: - description: "Public repository URL for package installation" - value: ${{ jobs.build_native_packages.outputs.package_repository_url }} - -permissions: - id-token: write - contents: read - -run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}, ${{ inputs.dist_amdgpu_families }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) - -jobs: - build_native_packages: - name: Build ${{ inputs.native_package_type }} packages - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - container: - image: ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest - outputs: - package_repository_url: ${{ steps.upload-packages.outputs.package_repository_url }} - env: - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - DIST_AMDGPU_FAMILIES: ${{ inputs.dist_amdgpu_families }} - PACKAGE_SUFFIX: ${{ inputs.package_suffix }} - # Use relative paths to avoid host/container path mismatch - OUTPUT_DIR: output - ARTIFACTS_DIR: output/artifacts - PACKAGE_DIST_DIR: output/packages - RELEASE_TYPE: ${{ inputs.release_type }} - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - - name: Install Python requirements - run: | - pip install pyelftools boto3 jinja2 pyzstd - - - name: Install System requirements - run: | - # Install the needed tools for creating rpm / deb packages - # Also install tools for creating repo files - sudo apt-get update - sudo apt-get install -y llvm-20 - sudo apt-get install -y rpm debhelper-compat build-essential - sudo apt-get install -y dpkg-dev createrepo-c - sudo apt-get install -y patchelf - - - name: Fetch Artifacts for all GPU families - run: | - # Use artifact_manager.py for consistent multi-arch artifact fetching - # NOTE: artifact_manager.py appends /artifacts to --output-dir in non-flatten mode, - # so we pass OUTPUT_DIR (parent) to have artifacts land in $OUTPUT_DIR/artifacts - # which matches the ARTIFACTS_DIR environment variable - python ./build_tools/artifact_manager.py fetch \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --stage=all \ - --amdgpu-families="${{ env.DIST_AMDGPU_FAMILIES }}" \ - --expand-family-to-targets \ - --output-dir="${{ env.OUTPUT_DIR }}" - - - name: Build Packages - id: build-packages - run: | - # GFX architectures are auto-detected from the artifact directory - # (populated by --expand-family-to-targets in the fetch step). - # KPACK_SPLIT_ARTIFACTS is auto-detected from therock_manifest.json. - python ./build_tools/packaging/linux/build_package.py \ - --dest-dir "${{ env.PACKAGE_DIST_DIR }}" \ - --rocm-version "${{ inputs.rocm_version }}" \ - --artifacts-dir "${{ env.ARTIFACTS_DIR }}" \ - --pkg-type "${{ inputs.native_package_type }}" \ - --version-suffix "${{ env.ARTIFACT_RUN_ID }}" \ - --build-variant "${{ inputs.build_variant }}" - - - name: Simulated install Test - id: test-packages - run: | - python ./build_tools/packaging/linux/native_linux_package_install_test.py \ - --test-type simulate \ - --packages-dir "${{ env.PACKAGE_DIST_DIR }}" \ - --pkg-type ${{ inputs.native_package_type }} - - - name: Configure AWS credentials for artifact uploads - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Upload Package repo to S3 - id: upload-packages - run: | - python ./build_tools/packaging/linux/upload_package_repo.py \ - --pkg-type "${{ inputs.native_package_type }}" \ - --run-id "${{ env.ARTIFACT_RUN_ID }}" \ - --package-dir "${{ env.PACKAGE_DIST_DIR }}" diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml deleted file mode 100644 index d0ddb8c93b..0000000000 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ /dev/null @@ -1,321 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-Arch Build - Sharded Pipeline for Linux -# -# This workflow builds TheRock in stages: -# 1. compiler-runtime (generic) - sysdeps, base, compiler, runtimes, profiler-core -# 2. runtime-tests (generic) - hip-tests, rocrtst (parallel to math-libs) -# 3. wsl-rocdxg (generic) - WSL-only ROCDXG bridge library (after compiler-runtime) -# 4. math-libs (per-arch) - BLAS, FFT, etc. -# 5. comm-libs (per-arch) - RCCL, rocshmem (parallel to math-libs) -# 6. storage-libs (generic) - hipfile (parallel to math-libs) -# 7. debug-tools (generic) - amd-dbgapi, rocr-debug-agent, rocgdb (parallel to math-libs) -# 8. dctools-core (generic) - RDC (parallel to math-libs) -# 9. profiler-apps (generic) - rocprofiler-systems (parallel to math-libs) -# 10. media-libs (generic) - sysdeps-amd-mesa, rocdecode, rocjpeg - -name: Multi-Arch Build (Linux) - -on: - workflow_call: - inputs: - artifact_group: - type: string - matrix_per_family_json: - type: string - description: "JSON array of {amdgpu_family, test-runs-on} objects for per-arch stages" - dist_amdgpu_families: - type: string - description: "Semicolon-separated list of all GPU families for dist targets" - build_variant_label: - type: string - build_variant_cmake_preset: - type: string - build_variant_suffix: - type: string - rocm_package_version: - type: string - prebuilt_stages: - type: string - default: "" - description: "Comma-separated build stages to skip (artifacts already copied by the orchestrator)" - build_runs_on: - type: string - default: "aws-linux-scale-rocm-prod-rc" - description: "Build runner label (selected by configure script with weighted distribution)" - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - external_repo_config: - description: "JSON config for external repo checkout (repository, ref, checkout_path, source_package, fetch_sources_args)" - type: string - default: "" - -permissions: - contents: read - - -jobs: - # ========================================================================== - # STAGE: compiler-runtime (generic) - # ========================================================================== - compiler-runtime: - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: compiler-runtime - stage_display_name: "Stage - Compiler Runtime" - timeout_minutes: 480 # 8 hours (compiler is big) - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: runtime-tests (generic, parallel to math-libs) - # ========================================================================== - runtime-tests: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: runtime-tests - stage_display_name: "Stage - Runtime Tests" - timeout_minutes: 120 # 2 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: wsl-rocdxg (generic) - # ========================================================================== - wsl-rocdxg: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'wsl-rocdxg') }} - uses: ./.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml - secrets: inherit - with: - stage_name: wsl-rocdxg - stage_display_name: "Stage - WSL ROCDXG" - timeout_minutes: 120 # 2 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: math-libs (per-arch) - # ========================================================================== - math-libs: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') }} - strategy: - # `fail-fast: false` lets all families complete even if one fails. - # This is useful for CI (to see all failures at once), but for releases - # this risks publishing incomplete packages. Release pipelines may want - # to either set `fail-fast: true` or add a validation step that blocks - # promotion unless all jobs succeeded. - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.matrix_per_family_json) }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: math-libs - stage_display_name: "Stage - Math Libs (${{ matrix.family_info.amdgpu_family }})" - timeout_minutes: 480 # 8 hours - amdgpu_family: ${{ matrix.family_info.amdgpu_family }} - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: comm-libs (generic, single job parallel to math-libs) - # RCCL and rocshmem are both TARGET_NEUTRAL artifacts. RCCL is built for - # Instinct/CDNA targets only (https://github.com/ROCm/TheRock/issues/2130). - # Running as a single generic job avoids redundant per-arch builds. - # ========================================================================== - comm-libs: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'comm-libs') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: comm-libs - stage_display_name: "Stage - Comm Libs" - timeout_minutes: 240 # 4 hours - amdgpu_family: "" - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: storage-libs (generic, single job parallel to math-libs) - # hipFile is a TARGET_NEUTRAL artifact. - # ========================================================================== - storage-libs: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'storage-libs') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: storage-libs - stage_display_name: "Stage - Storage Libs" - timeout_minutes: 240 # 4 hours - amdgpu_family: "" - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: debug-tools (generic, parallel to math-libs) - # ========================================================================== - debug-tools: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: debug-tools - stage_display_name: "Stage - Debug Tools" - timeout_minutes: 180 # 3 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: dctools-core (generic, parallel to math-libs) - # ========================================================================== - dctools-core: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'dctools-core') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: dctools-core - stage_display_name: "Stage - DC Tools Core" - timeout_minutes: 120 # 2 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: profiler-apps (generic, parallel to math-libs) - # ========================================================================== - profiler-apps: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'profiler-apps') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: profiler-apps - stage_display_name: "Stage - Profiler Apps" - timeout_minutes: 180 # 3 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: media-libs (generic) - # ========================================================================== - media-libs: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'media-libs') }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: media-libs - stage_display_name: "Stage - Media Libs" - timeout_minutes: 180 # 3 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml deleted file mode 100644 index 85ef04d8fd..0000000000 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable Workflow: Build Stage Artifacts -# -# This workflow builds TheRock stage artifacts for all stages. -# Handles both generic stages (compiler-runtime, profiler-apps, etc.) and -# per-arch stages (math-libs, comm-libs) using conditional logic. -# -# Artifacts flow between stages via S3 using the artifact_manager.py tool. -# -# TODO: When kpack splitting is permanently enabled, change fetch commands to use -# --generic-only instead of --amdgpu-families for efficiency. Subsequent build -# stages only need generic (host) artifacts; device-specific kpacks are not -# needed until final packaging. - -name: Build Stage Artifacts - -on: - workflow_call: - inputs: - stage_name: - type: string - required: true - description: "Stage name (e.g., compiler-runtime, math-libs)" - stage_display_name: - type: string - required: true - description: "Human-readable display name for the job" - timeout_minutes: - type: number - required: true - description: "Job timeout in minutes" - amdgpu_family: - type: string - required: false - default: "" - description: "GPU family for per-arch stages (empty for generic stages)" - dist_amdgpu_families: - type: string - required: true - description: "Semicolon-separated list of all GPU families for dist targets" - rocm_package_version: - type: string - required: true - description: "ROCm package version string" - build_variant_cmake_preset: - type: string - required: false - default: "" - description: "CMake preset name for this build variant (optional)" - release_type: - description: 'Release type; controls artifact bucket and IAM role' - type: string - default: ci - build_runs_on: - description: "Build runner label (selected by configure script with weighted distribution)" - type: string - default: "aws-linux-scale-rocm-prod-rc" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - external_repo_config: - description: "JSON config for external repo checkout (generated by detect_external_repo_config.py)" - type: string - default: "" - -permissions: - contents: read - -jobs: - build_stage: - name: ${{ inputs.stage_display_name }} - runs-on: ${{ inputs.build_runs_on }} - timeout-minutes: ${{ inputs.timeout_minutes }} - permissions: - contents: read - id-token: write - container: - image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - env: - STAGE_NAME: ${{ inputs.stage_name }} - AMDGPU_FAMILIES: ${{ inputs.amdgpu_family }} - BUILD_DIR: build - CCACHE_CONFIGPATH: ${{ github.workspace }}/.ccache/ccache.conf - CACHE_DIR: ${{ github.workspace }}/.container-cache - TEATIME_FORCE_INTERACTIVE: 0 - RELEASE_TYPE: ${{ inputs.release_type }} - EXTERNAL_FETCH_SOURCES_ARGS: ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).fetch_sources_args || '' }} - - steps: - - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout external repository - if: ${{ inputs.external_repo_config != '' }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ fromJSON(inputs.external_repo_config).repository }} - ref: ${{ fromJSON(inputs.external_repo_config).ref }} - path: ${{ fromJSON(inputs.external_repo_config).checkout_path }} - - - name: Install python deps - run: pip install -r requirements.txt - - - name: Adjust git config - run: | - git config --global --add safe.directory $PWD - git config fetch.parallel 10 - - - name: Setup ccache - run: | - ./build_tools/setup_ccache.py \ - --release-type "${{ inputs.release_type }}" \ - --dir "$(dirname $CCACHE_CONFIGPATH)" \ - --local-path "$CACHE_DIR/ccache" - - - name: Runner health status - run: | - ./build_tools/health_status.py - - # Note: no AWS credentials should be needed here since our CI buckets - # all currently have public read access. - - name: Fetch inbound artifacts - run: | - python build_tools/artifact_manager.py fetch \ - --run-id=${{ github.run_id }} \ - --stage="${STAGE_NAME}" \ - --amdgpu-families="${{ inputs.amdgpu_family }}" \ - --output-dir="${BUILD_DIR}" \ - --bootstrap - - - name: Fetch sources - timeout-minutes: 30 - run: ./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1 ${EXTERNAL_FETCH_SOURCES_ARGS} - - - name: Get stage configuration - id: stage_config - run: | - python build_tools/configure_stage.py \ - --stage="${STAGE_NAME}" \ - --amdgpu-families="${{ inputs.amdgpu_family }}" \ - --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ - --manylinux \ - --gha-output - - - name: Install stage python deps - if: ${{ steps.stage_config.outputs.pip_install_cmd }} - run: pip install ${{ steps.stage_config.outputs.pip_install_cmd }} - - - name: Configure - run: | - cmake -B "${BUILD_DIR}" -S . -GNinja \ - -DTHEROCK_PACKAGE_VERSION=${{ inputs.rocm_package_version }} \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - ${{ inputs.build_variant_cmake_preset && format('--preset={0}', inputs.build_variant_cmake_preset) }} \ - ${{ inputs.external_repo_config != '' && format('-DTHEROCK_{0}_SOURCE_DIR={1}', fromJSON(inputs.external_repo_config).source_package, fromJSON(inputs.external_repo_config).checkout_path) || '' }} \ - ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).extra_cmake_options || '' }} \ - ${{ steps.stage_config.outputs.cmake_args }} - - - name: Build stage - run: | - cmake --build "${BUILD_DIR}" --target stage-${STAGE_NAME} therock-artifacts -- -k 0 - - - name: Generate failed log summaries - if: ${{ !cancelled() }} - run: | - python3 build_tools/github_actions/detect_failed_logs.py - - - name: Run build tests (Comgr, blocking) - if: ${{ !cancelled() && inputs.stage_name == 'compiler-runtime' }} - env: - TEATIME_FORCE_INTERACTIVE: 1 - run: | - cmake --build "${BUILD_DIR}" --target build-test-amd-comgr -- -k 0 - - # Remaining build tests (e.g. rocjitsu, mirage) still run and report, but - # are not yet gating. Comgr is split out above so its failures block CI. - # TODO(#6112): rocjitsu tests are being moved out; drop this step once done. - - name: Run build tests (other, non-blocking) - if: ${{ !cancelled() && inputs.stage_name == 'compiler-runtime' }} - continue-on-error: true - env: - TEATIME_FORCE_INTERACTIVE: 1 - run: | - cmake --build "${BUILD_DIR}" --target therock-build-tests -- -k 0 - - - name: Report - if: ${{ !cancelled() }} - run: | - echo "CCache Stats:" - ccache -s -v - echo "Artifacts:" - ls -lh "${BUILD_DIR}"/artifacts/*.tar.xz 2>/dev/null || echo "No artifacts found" - - - name: Test Packaging - run: | - ctest --test-dir ${BUILD_DIR} --output-on-failure - - - name: Configure AWS credentials for artifact uploads - if: ${{ always() }} - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - # TODO(#3336): Attempt upload `always()`? - # * If some part of the build failed, partial artifacts may be useful for debugging. - # * This script fails if `build_dir / "artifacts"` does not exist. That's useful - # if the build succeeded but did not produce artifacts. That's _not_ useful - # if the workflow failed or was cancelled before the build even started though. - - name: Push stage artifacts - run: | - python build_tools/artifact_manager.py push \ - --run-id ${{ github.run_id }} \ - --stage="${STAGE_NAME}" \ - --build-dir="${BUILD_DIR}" - - - name: Upload stage logs - if: ${{ always() }} - run: | - python build_tools/github_actions/post_stage_upload.py \ - --run-id=${{ github.run_id }} \ - --stage="${STAGE_NAME}" \ - --build-dir="${BUILD_DIR}" \ - --amdgpu-family="${AMDGPU_FAMILIES}" diff --git a/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels.yml b/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels.yml deleted file mode 100644 index c8596b34e3..0000000000 --- a/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels.yml +++ /dev/null @@ -1,501 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable multi-arch PyTorch build workflow for Linux. -# -# Built shape: -# - One fat build_prod_wheels.py invocation for one python_version and -# pytorch_git_ref, covering all gfx targets expanded from the -# inputs.amdgpu_families set. Builds torch, torchaudio, torchvision, -# triton, and apex in a single invocation. -# - kpack-split the fat torch and torchvision wheels into host wheels -# plus per-gfx amd-torch-device-* / amd-torchvision-device-* wheels. -# other packages ship as-is (nothing to split). -# - Upload the result to s3://therock-{release_type}-python/v4/whl/ -# via publish_pytorch_to_release_bucket.py (pip-compatible index generation -# happens server-side). -# -# For convergence with CI and testing plans see: -# * https://github.com/ROCm/TheRock/issues/3332 -# * https://github.com/ROCm/TheRock/issues/3291 - -name: Multi-Arch Build Portable Linux PyTorch Wheels - -on: - workflow_call: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx94X-dcgpu;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - test_amdgpu_families: - description: >- - Semicolon-separated AMD GPU families to run tests for (e.g. - 'gfx94X-dcgpu'). Use 'auto' to test built families, or 'none' to skip - tests. - type: string - default: "auto" - python_version: - description: Python version to build wheels for. - type: string - required: true - pytorch_git_ref: - description: PyTorch ref to checkout. Typically "nightly" or "release/X.Y". - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; selects the S3 bucket.' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type ('none', 'sccache', or 'ccache')" - type: string - default: "sccache" - run_full_pytorch_tests: - description: >- - Dispatch the full PyTorch test suite after a successful build. - Only takes effect for nightly gfx94X-dcgpu Python 3.12 builds. - type: boolean - default: false - workflow_dispatch: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx94X-dcgpu;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - test_amdgpu_families: - description: >- - Semicolon-separated AMD GPU families to run tests for (e.g. - 'gfx94X-dcgpu'). Use 'auto' to test built families, or 'none' to skip - tests. - type: string - default: "auto" - python_version: - description: Python version to build wheels for (for example, "3.12"). - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (for example, "release/2.12"). - type: string - default: "release/2.12" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - run_full_pytorch_tests: - description: >- - Dispatch the full PyTorch test suite after a successful build. - Only takes effect for nightly gfx94X-dcgpu Python 3.12 builds. - type: boolean - default: false - -run-name: Build Multi-Arch Linux PyTorch Wheels (${{ inputs.release_type }}, ${{ inputs.python_version }}, ${{ inputs.pytorch_git_ref }}) - -permissions: - contents: read - -jobs: - build_pytorch_wheels: - name: Build | py ${{ inputs.python_version }} | torch ${{ inputs.pytorch_git_ref }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - permissions: - id-token: write - contents: read - container: - image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - env: - PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist - optional_build_prod_arguments: "" - SCCACHE_BUCKET: "therock-pytorch-sccache-${{ inputs.release_type }}" - SCCACHE_REGION: us-east-1 - SCCACHE_S3_KEY_PREFIX: "linux/multiarch/v1/" - SCCACHE_S3_USE_SSL: "true" - SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "true" - SCCACHE_IDLE_TIMEOUT: "0" # never idle-out; keeps stats accurate on long builds - SCCACHE_LOG: warn - outputs: - torch_version: ${{ steps.pytorch-versions.outputs.torch_version }} - torchaudio_version: ${{ steps.pytorch-versions.outputs.torchaudio_version }} - apex_version: ${{ steps.pytorch-versions.outputs.apex_version }} - torchvision_version: ${{ steps.pytorch-versions.outputs.torchvision_version }} - triton_version: ${{ steps.pytorch-versions.outputs.triton_version }} - package_index_url: ${{ steps.publish-pytorch-wheels.outputs.package_index_url }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - name: Select Python version - run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} - - - name: Add selected Python version to PATH - run: | - python_dir="/opt/python/${{ env.cp_version }}" - if ! [ -x "${python_dir}/bin/python" ]; then - echo "ERROR: Could not find python: ${python_dir}" - exit 1 - fi - echo "${python_dir}/bin" >> "$GITHUB_PATH" - - - name: Install python deps for CI - run: | - pip install -r external-builds/pytorch/requirements-ci.txt - - - name: Checkout PyTorch source repos (nightly) - if: ${{ inputs.pytorch_git_ref == 'nightly' }} - run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --repo-hashtag nightly \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_audio_repo.py checkout \ - --repo-hashtag nightly \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_vision_repo.py checkout \ - --repo-hashtag nightly \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_triton_repo.py checkout \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_apex_repo.py checkout \ - --repo-hashtag master \ - --no-commit-hipify - - - name: Checkout PyTorch source repos (stable) - if: ${{ inputs.pytorch_git_ref != 'nightly' }} - run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --gitrepo-origin https://github.com/ROCm/pytorch.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_audio_repo.py checkout \ - --require-related-commit \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_vision_repo.py checkout \ - --require-related-commit \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_triton_repo.py checkout \ - --no-commit-hipify - ./external-builds/pytorch/pytorch_apex_repo.py checkout \ - --require-related-commit \ - --no-commit-hipify - - # determine_version.py sets optional_build_prod_arguments in - # GITHUB_ENV (--rocm-sdk-version, --version-suffix). - - name: Determine optional arguments passed to `build_prod_wheels.py` - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version ${{ inputs.rocm_version }} - - - name: Configure AWS Credentials for sccache - if: ${{ inputs.cache_type == 'sccache' && github.repository_owner == 'ROCm' }} - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::324352301041:role/therock-${{ inputs.release_type }} - # Multi-package PyTorch builds can exceed the default 1h credential - # window; the role's MaxSessionDuration must be >= this value. - role-duration-seconds: 21600 - - - name: Verify sccache - if: ${{ inputs.cache_type == 'sccache' }} - run: | - echo "sccache version: $(sccache --version)" - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" - echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}" - - - name: Expand amdgpu families to gfx targets - id: expand - run: | - targets=$(python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_families }}") - echo "Expanded families '${{ inputs.amdgpu_families }}' -> '${targets}'" - echo "targets=${targets}" >> "$GITHUB_OUTPUT" - - - name: Build PyTorch wheels - run: | - cache_flag="" - if [ "${{ inputs.cache_type }}" = "sccache" ]; then - cache_flag="--use-sccache" - elif [ "${{ inputs.cache_type }}" = "ccache" ]; then - cache_flag="--use-ccache" - fi - - # build_prod_wheels.py forwards each of these to pip and ignores an - # empty value, so pass both and supply whichever source applies: - # --index-url for a PEP 503 index, --find-links for a flat page. - ./external-builds/pytorch/build_prod_wheels.py \ - build \ - --install-rocm \ - --index-url="${{ inputs.rocm_package_index_url }}" \ - --find-links="${{ inputs.rocm_package_find_links_url }}" \ - --clean \ - --output-dir ${{ env.PACKAGE_DIST_DIR }} \ - ${cache_flag} \ - --pytorch-rocm-arch ${{ steps.expand.outputs.targets }} \ - ${{ env.optional_build_prod_arguments }} - - - name: Report cache stats - if: ${{ !cancelled() && inputs.cache_type != 'none' }} - run: | - if [ "${{ inputs.cache_type }}" = "sccache" ]; then - echo "sccache stats:" - echo "--------------" - sccache --show-stats || true - elif [ "${{ inputs.cache_type }}" = "ccache" ]; then - echo "ccache stats:" - echo "-------------" - ccache -s -v || true - fi - - - name: Sanity check wheel - run: | - python external-builds/pytorch/sanity_check_wheel.py \ - ${{ env.PACKAGE_DIST_DIR }}/ - - # The kpack wheel splitter takes the fat torch wheel and produces a - # smaller host wheel plus one device wheel per gfx target. Source-install - # from the TheRock checkout - kpack has no published wheel today. - # The outer actions/checkout skips submodules for speed, so pull in - # rocm-systems only when we actually need it. - - name: Init rocm-systems submodule - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --depth=1 rocm-systems - - - name: Install kpack tooling - run: | - pip install \ - -e rocm-systems/python/rocm-bootstrap \ - -e rocm-systems/shared/kpack - - - name: Split fat wheels (torch + torchvision) - run: | - # The splitter writes the host wheel under the same filename as - # the input, so relocate each fat wheel out of PACKAGE_DIST_DIR - # before invoking the splitter. torchaudio, triton, and apex ship - # as-is: torchaudio and triton have no fat binaries to split, and - # apex's ROCm fork ships JIT-loader shims rather than precompiled - # extensions, so its wheel is constant-size in arch count. - FAT_STAGE="${RUNNER_TEMP:-/tmp}/pytorch-fat" - mkdir -p "${FAT_STAGE}" - - ROCM_SDK_VERSION="$(python -m rocm_sdk version)" - - mv ${{ env.PACKAGE_DIST_DIR }}/torch-*.whl "${FAT_STAGE}/" - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torch-*.whl \ - --output-dir "${{ env.PACKAGE_DIST_DIR }}" \ - --device-package-prefix amd-torch-device \ - --overlay-root torch/ \ - --wheel-type torch-fat \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j "$(nproc)" - rm -f "${FAT_STAGE}"/torch-*.whl - - mv ${{ env.PACKAGE_DIST_DIR }}/torchvision-*.whl "${FAT_STAGE}/" - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torchvision-*.whl \ - --output-dir "${{ env.PACKAGE_DIST_DIR }}" \ - --device-package-prefix amd-torchvision-device \ - --overlay-root torchvision/ \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j "$(nproc)" - - rm -rf "${FAT_STAGE}" - echo "Post-split contents of ${{ env.PACKAGE_DIST_DIR }}:" - ls -lh "${{ env.PACKAGE_DIST_DIR }}/" - - # TODO(#5110): Once manifest-driven builds land, write these outputs from - # the manifest and keep wheel scanning as a validation step that the build - # produced the requested package versions. - - name: Write PyTorch package version outputs - id: pytorch-versions - run: | - python build_tools/github_actions/write_torch_versions.py \ - --dist-dir ${{ env.PACKAGE_DIST_DIR }} - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Upload PyTorch wheels to release bucket - id: publish-pytorch-wheels - run: | - python build_tools/github_actions/publish_pytorch_to_release_bucket.py \ - --source-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --release-type="${{ inputs.release_type }}" - - configure_pytorch_tests: - name: Configure PyTorch Tests - runs-on: ubuntu-24.04 - permissions: - contents: read - outputs: - enabled: ${{ steps.configure.outputs.enabled }} - matrix: ${{ steps.configure.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - - name: Configure PyTorch test matrix - id: configure - env: - CI_CONFIG_PATH: ci-config - run: | - python build_tools/github_actions/configure_pytorch_test_matrix.py \ - --build-amdgpu-families "${{ inputs.amdgpu_families }}" \ - --test-amdgpu-families "${{ inputs.test_amdgpu_families }}" \ - --platform linux - - test_pytorch_wheels: - name: Test | ${{ matrix.amdgpu_family }} | ${{ matrix.test_runs_on }} - if: ${{ needs.configure_pytorch_tests.outputs.enabled == 'true' }} - needs: [build_pytorch_wheels, configure_pytorch_tests] - permissions: - contents: read - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.configure_pytorch_tests.outputs.matrix) }} - uses: ./.github/workflows/test_pytorch_wheels.yml - with: - amdgpu_family: ${{ matrix.amdgpu_family }} - test_runs_on: ${{ matrix.test_runs_on }} - package_index_url: ${{ needs.build_pytorch_wheels.outputs.package_index_url }} - python_version: ${{ inputs.python_version }} - torch_version: ${{ needs.build_pytorch_wheels.outputs.torch_version }} - # TODO(#5110): pass manifest_url here and let test_pytorch_wheels.yml use - # checkout_from_manifest.py --no-hipify --no-submodules so tests use the - # exact source commit that produced the package under test. - pytorch_git_ref: ${{ inputs.pytorch_git_ref }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - # Dispatch the full PyTorch test suite for this exact build. Because this - # workflow builds a single (python_version, pytorch_git_ref), the build job's - # torch_version output is unambiguous and only this build's success gates the - # dispatch -- a failure for one ref no longer skips testing the others (#5777). - # Scoped to nightly gfx94X-dcgpu Python 3.12 builds for now. - dispatch_pytorch_wheels_full_test: - name: Dispatch PyTorch Full Test | torch ${{ inputs.pytorch_git_ref }} - if: >- - ${{ - inputs.run_full_pytorch_tests && - inputs.release_type == 'nightly' && - inputs.python_version == '3.12' && - contains(inputs.amdgpu_families, 'gfx94X-dcgpu') - }} - needs: [build_pytorch_wheels] - runs-on: ubuntu-24.04 - permissions: - actions: write - contents: read - env: - PYTORCH_GIT_REF: ${{ inputs.pytorch_git_ref }} - steps: - - name: Check full test cadence - id: cadence - # Release branches test daily; the nightly branch tests weekly (Sunday). - run: | - if [ "${PYTORCH_GIT_REF}" != "nightly" ] || [ "$(date -u +%u)" = "7" ]; then - echo "dispatch=true" >> "$GITHUB_OUTPUT" - else - echo "dispatch=false" >> "$GITHUB_OUTPUT" - fi - - - name: Dispatch full PyTorch test workflow - if: ${{ steps.cadence.outputs.dispatch == 'true' }} - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: test_pytorch_wheels_full.yml - inputs: | - { "amdgpu_family": "gfx94X-dcgpu", - "test_runs_on": "linux-gfx942-1gpu-ossci-rocm", - "test_runs_on_multi_gpu": "linux-gfx942-8gpu-ossci-rocm", - "package_index_url": "${{ needs.build_pytorch_wheels.outputs.package_index_url }}", - "python_version": "${{ inputs.python_version }}", - "torch_version": "${{ needs.build_pytorch_wheels.outputs.torch_version }}", - "pytorch_git_ref": "${{ inputs.pytorch_git_ref }}", - "pytorch_git_repo": "${{ inputs.pytorch_git_ref == 'nightly' && 'pytorch/pytorch' || 'ROCm/pytorch' }}", - "test_configs": "default distributed inductor", - "tests_to_include": "", - "repository": "${{ inputs.repository || github.repository }}", - "ref": "${{ inputs.ref }}" - } diff --git a/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels_ci.yml b/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels_ci.yml deleted file mode 100644 index 59bafa46f1..0000000000 --- a/.github/workflows/multi_arch_build_portable_linux_pytorch_wheels_ci.yml +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# CI variant of multi_arch_build_portable_linux_pytorch_wheels.yml. -# -# Key differences from the release workflow: -# - Installs ROCm packages from either a flat find-links page (CI artifacts, -# via rocm_package_find_links_url) or a PEP 503 index (via -# rocm_package_index_url); provide whichever one applies. -# - Builds torch only (no torchvision, torchaudio, or triton) -# - No S3 upload, staging, testing, or promotion — just build + sanity check -# - Uses sccache with S3-backed storage (configurable via cache_type input) -# -# TODO(#3291): Build more packages (torchvision, torchaudio, triton, etc.) -# -# Both workflows share build_prod_wheels.py for the actual build logic. -# See https://github.com/ROCm/TheRock/issues/3291 for convergence plans. - -name: Build Portable Linux PyTorch Wheels (CI) - -on: - workflow_call: - inputs: - artifact_group: - type: string - required: true - python_version: - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (typically "release/X.Y") - type: string - default: "release/2.11" - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - cache_type: - description: "Compiler cache type (sccache, ccache, or none)" - required: false - type: string - default: "sccache" - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx94X-dcgpu;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. Leave empty - for legacy callers that rely on build_prod_wheels.py's - `rocm-sdk targets` fallback. - type: string - default: "" - kpack_split: - description: "'true' if this is a kpack-split flat build." - type: string - default: "false" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - workflow_dispatch: - inputs: - artifact_group: - description: "The artifact group to build (e.g. gfx94X-dcgpu, gfx120X-all)" - type: string - default: gfx94X-dcgpu - python_version: - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (typically "release/X.Y") - type: string - default: "release/2.11" - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - -permissions: - contents: read - -run-name: Build portable Linux PyTorch Wheels CI (${{ inputs.artifact_group }}, py${{ inputs.python_version }}, ${{ inputs.pytorch_git_ref }}) - -jobs: - build_pytorch_wheels: - name: Build PyTorch | ${{ inputs.artifact_group }} | torch ${{ inputs.pytorch_git_ref }} | py${{ inputs.python_version }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - permissions: - id-token: write - contents: read - container: - image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - env: - PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist - RELEASE_TYPE: ci - optional_build_prod_arguments: "" - SCCACHE_BUCKET: therock-pytorch-sccache-ci - SCCACHE_REGION: us-east-1 - SCCACHE_S3_KEY_PREFIX: "linux/${{ inputs.artifact_group }}/" - SCCACHE_S3_USE_SSL: "true" - SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "true" - SCCACHE_IDLE_TIMEOUT: "0" # never idle-out; keeps stats accurate on long builds - SCCACHE_LOG: warn - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - name: Select Python version - run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} - - - name: Add selected Python version to PATH - run: | - python_dir="/opt/python/${{ env.cp_version }}" - if ! [ -x "${python_dir}/bin/python" ]; then - echo "ERROR: Could not find python: ${python_dir}" - exit 1 - fi - echo "${python_dir}/bin" >> "$GITHUB_PATH" - - - name: Install python deps for CI - run: | - pip install -r external-builds/pytorch/requirements-ci.txt - - - name: Checkout PyTorch source (nightly) - if: ${{ inputs.pytorch_git_ref == 'nightly' }} - run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --repo-hashtag nightly \ - --no-commit-hipify - - - name: Checkout PyTorch source (stable) - if: ${{ inputs.pytorch_git_ref != 'nightly' }} - run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --gitrepo-origin https://github.com/ROCm/pytorch.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-commit-hipify - - # Note: determine_version.py sets optional_build_prod_arguments in - # GITHUB_ENV, which includes --rocm-sdk-version and --version-suffix. - - name: Determine optional arguments passed to `build_prod_wheels.py` - if: ${{ inputs.rocm_version }} - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version ${{ inputs.rocm_version }} - - # sccache uses an S3 bucket reached via an OIDC-assumed IAM role, which - # fork PRs / external repos cannot assume. Compute the effective cache - # type once (downgrading sccache -> none on forks) so every sccache step - # below keys off a single decision instead of repeating fork checks. - - name: Determine cache type - id: cache - run: | - python build_tools/github_actions/compute_pytorch_cache_type.py \ - --cache-type "${{ inputs.cache_type }}" - - - name: Configure AWS Credentials for sccache - if: ${{ steps.cache.outputs.cache_type == 'sccache' }} - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::324352301041:role/therock-ci - # Multi-package PyTorch builds can exceed the default 1h credential - # window; the role's MaxSessionDuration must be >= this value. - role-duration-seconds: 21600 - - - name: Verify sccache - if: ${{ steps.cache.outputs.cache_type == 'sccache' }} - run: | - echo "sccache version: $(sccache --version)" - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" - echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}" - - - name: Expand amdgpu families to gfx targets - if: ${{ inputs.amdgpu_families != '' }} - id: expand - run: | - targets=$(python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_families }}") - echo "Expanded families '${{ inputs.amdgpu_families }}' -> '${targets}'" - echo "targets=${targets}" >> "$GITHUB_OUTPUT" - - # inputs.amdgpu_families is only populated by the multi-arch caller - # today. Legacy single-family CI (ci_linux.yml) leaves it empty and - # falls back to `rocm-sdk targets` inside build_prod_wheels.py. - # TODO(#4687): Make --pytorch-rocm-arch required here once every - # caller plumbs the value through and the fallback in - # build_prod_wheels.py::get_rocm_sdk_targets is removed. - - name: Build PyTorch wheels - run: | - cache_flag="" - if [ "${{ steps.cache.outputs.cache_type }}" = "sccache" ]; then - cache_flag="--use-sccache" - elif [ "${{ steps.cache.outputs.cache_type }}" = "ccache" ]; then - cache_flag="--use-ccache" - fi - - pytorch_rocm_arch_flag="" - if [ -n "${{ steps.expand.outputs.targets }}" ]; then - pytorch_rocm_arch_flag="--pytorch-rocm-arch ${{ steps.expand.outputs.targets }}" - fi - - # build_prod_wheels.py forwards each of these to pip and ignores an - # empty value, so pass both and supply whichever source applies: - # --index-url for a PEP 503 index, --find-links for a flat page. - ./external-builds/pytorch/build_prod_wheels.py \ - build \ - --install-rocm \ - --index-url="${{ inputs.rocm_package_index_url }}" \ - --find-links="${{ inputs.rocm_package_find_links_url }}" \ - --clean \ - --output-dir ${{ env.PACKAGE_DIST_DIR }} \ - ${cache_flag} \ - ${pytorch_rocm_arch_flag} \ - ${{ env.optional_build_prod_arguments }} - - - name: Report cache stats - if: ${{ !cancelled() && steps.cache.outputs.cache_type != 'none' }} - run: | - if [ "${{ steps.cache.outputs.cache_type }}" = "sccache" ]; then - echo "sccache stats:" - echo "--------------" - sccache --show-stats || true - elif [ "${{ steps.cache.outputs.cache_type }}" = "ccache" ]; then - echo "ccache stats:" - echo "-------------" - ccache -s -v || true - fi - - - name: Sanity check wheel - run: | - python external-builds/pytorch/sanity_check_wheel.py \ - ${{ env.PACKAGE_DIST_DIR }}/ - - # The kpack wheel splitter takes the fat torch wheel and produces a - # smaller host wheel plus one device wheel per gfx target. Source-install - # from the TheRock checkout — kpack has no published wheel today. - # The outer actions/checkout skips submodules for speed, so pull in - # rocm-systems only when we actually need it. - - name: Init rocm-systems submodule - if: ${{ inputs.kpack_split == 'true' }} - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --depth=1 rocm-systems - - - name: Install kpack tooling - if: ${{ inputs.kpack_split == 'true' }} - run: | - pip install \ - -e rocm-systems/python/rocm-bootstrap \ - -e rocm-systems/shared/kpack - - - name: Split PyTorch fat wheel - if: ${{ inputs.kpack_split == 'true' }} - run: | - # The splitter writes the host wheel under the same filename as the - # input, so relocate the fat wheel out of PACKAGE_DIST_DIR first. - FAT_STAGE="${RUNNER_TEMP:-/tmp}/pytorch-fat" - mkdir -p "${FAT_STAGE}" - mv ${{ env.PACKAGE_DIST_DIR }}/torch-*.whl "${FAT_STAGE}/" - - ROCM_SDK_VERSION="$(python -m rocm_sdk version)" - - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torch-*.whl \ - --output-dir "${{ env.PACKAGE_DIST_DIR }}" \ - --device-package-prefix amd-torch-device \ - --overlay-root torch/ \ - --wheel-type torch-fat \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j "$(nproc)" - - rm -rf "${FAT_STAGE}" - echo "Post-split contents of ${{ env.PACKAGE_DIST_DIR }}:" - ls -lh "${{ env.PACKAGE_DIST_DIR }}/" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ci - - # Appends to the same therock-ci-artifacts/-linux/python/ prefix - # that the ROCm python job already populated — one find-links URL - # covers the whole SDK + PyTorch stack. - - name: Upload PyTorch wheels - run: | - python build_tools/github_actions/upload_python_packages.py \ - --input-packages-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --run-id="${{ github.run_id }}" \ - --multiarch diff --git a/.github/workflows/multi_arch_build_tarballs.yml b/.github/workflows/multi_arch_build_tarballs.yml deleted file mode 100644 index 6aab019b68..0000000000 --- a/.github/workflows/multi_arch_build_tarballs.yml +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Build Multi-Arch Tarballs - -on: - workflow_dispatch: - inputs: - artifact_run_id: - description: "Run ID to fetch artifacts from (defaults to current run)" - type: string - default: "" - artifact_github_repo: - description: "GitHub repository for artifact_run_id" - type: string - default: ROCm/TheRock - dist_amdgpu_families: - description: "Semicolon-separated list of GPU families (e.g. 'gfx94X-dcgpu;gfx110X-all')" - type: string - platform: - type: choice - description: "Platform to fetch artifacts for" - options: - - linux - - windows - default: "linux" - package_version: - description: "ROCm package version string (e.g. '7.13.0.dev0+abc123')" - type: string - release_type: - description: 'Release type: "ci" or "dev"' - type: choice - options: - - ci - - dev - default: ci - include_test_tarballs: - description: "Also build -tests tarballs that include test artifacts" - type: boolean - default: true - workflow_call: - inputs: - artifact_run_id: - type: string - default: "" - artifact_github_repo: - type: string - default: "" - dist_amdgpu_families: - type: string - required: true - platform: - type: string - default: "linux" - package_version: - type: string - required: true - release_type: - type: string - default: ci - include_test_tarballs: - type: boolean - default: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - outputs: - tarball_urls: - description: Tarball URLs keyed by family - value: ${{ jobs.build_tarballs.outputs.tarball_urls }} - -permissions: - contents: read - -run-name: Build Multi-Arch Tarballs (${{ inputs.dist_amdgpu_families }}, ${{ inputs.platform }}) - -jobs: - build_tarballs: - name: Build Tarballs - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - container: - image: python:3.12-slim - permissions: - id-token: write - outputs: - tarball_urls: ${{ steps.upload.outputs.tarball_urls }} - env: - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - RELEASE_TYPE: ${{ inputs.release_type }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || github.ref_name }} - - - name: Install Python requirements - run: pip install -r requirements.txt - - - name: Build tarballs - shell: bash - run: | - EXTRA_ARGS=() - if [[ "${{ inputs.include_test_tarballs }}" == "true" ]]; then - EXTRA_ARGS+=(--include-test-tarballs) - fi - - python build_tools/build_tarballs.py \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ - --platform="${{ inputs.platform }}" \ - --package-version="${{ inputs.package_version }}" \ - --output-dir="${{ github.workspace }}/tarballs" \ - "${EXTRA_ARGS[@]}" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Upload tarballs - id: upload - run: | - python build_tools/github_actions/upload_tarballs.py \ - --input-tarballs-dir="${{ github.workspace }}/tarballs" \ - --run-id="${{ github.run_id }}" \ - --platform="${{ inputs.platform }}" \ - --release-type="${{ inputs.release_type }}" diff --git a/.github/workflows/multi_arch_build_windows.yml b/.github/workflows/multi_arch_build_windows.yml deleted file mode 100644 index 4c109bda54..0000000000 --- a/.github/workflows/multi_arch_build_windows.yml +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-Arch Build - Sharded Pipeline for Windows -# -# This workflow builds TheRock in stages: -# 1. compiler-runtime (generic) - sysdeps, base, compiler, runtimes -# 2. runtime-tests (generic) - hip-tests (parallel to math-libs) -# 3. math-libs (per-arch) - BLAS, FFT, etc. -# 4. debug-tools (generic) - amd-dbgapi (rocgdb eventually). -# -# Stages disabled on Windows: comm-libs, dctools-core, profiler-apps, media -# (see BUILD_TOPOLOGY.toml disable_platforms) -# -# Artifacts flow between stages via S3 using the artifact_manager.py tool. - -name: Multi-Arch Build (Windows) - -on: - workflow_call: - inputs: - artifact_group: - type: string - matrix_per_family_json: - type: string - description: "JSON array of {amdgpu_family, test-runs-on} objects for per-arch stages" - dist_amdgpu_families: - type: string - description: "Semicolon-separated list of all GPU families for dist targets" - build_variant_label: - type: string - build_variant_cmake_preset: - type: string - build_variant_suffix: - type: string - prebuilt_stages: - type: string - default: "" - description: "Comma-separated build stages to skip (artifacts already copied by the orchestrator)" - build_runs_on: - type: string - default: "azure-windows-scale-rocm" - description: "Build runner label (selected by configure script with weighted distribution)" - rocm_package_version: - type: string - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - external_repo_config: - description: "JSON config for external repo checkout (repository, ref, checkout_path, source_package)" - type: string - default: "" - -permissions: - contents: read - -jobs: - # ========================================================================== - # STAGE: compiler-runtime (generic) - # ========================================================================== - compiler-runtime: - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') }} - uses: ./.github/workflows/multi_arch_build_windows_artifacts.yml - secrets: inherit - with: - stage_name: compiler-runtime - stage_display_name: "Stage - Compiler Runtime" - timeout_minutes: 480 # 8 hours (compiler is big) - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: runtime-tests (generic, parallel to math-libs) - # ========================================================================== - runtime-tests: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') }} - uses: ./.github/workflows/multi_arch_build_windows_artifacts.yml - secrets: inherit - with: - stage_name: runtime-tests - stage_display_name: "Stage - Runtime Tests" - timeout_minutes: 120 # 2 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: math-libs (per-arch) - # ========================================================================== - math-libs: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') }} - strategy: - # `fail-fast: false` lets all families complete even if one fails. - # This is useful for CI (to see all failures at once), but for releases - # this risks publishing incomplete packages. Release pipelines may want - # to either set `fail-fast: true` or add a validation step that blocks - # promotion unless all jobs succeeded. - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.matrix_per_family_json) }} - uses: ./.github/workflows/multi_arch_build_windows_artifacts.yml - secrets: inherit - with: - stage_name: math-libs - stage_display_name: "Stage - Math Libs (${{ matrix.family_info.amdgpu_family }})" - timeout_minutes: 480 # 8 hours - amdgpu_family: ${{ matrix.family_info.amdgpu_family }} - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write - - # ========================================================================== - # STAGE: debug-tools (generic, parallel to math-libs) - # ========================================================================== - debug-tools: - needs: compiler-runtime - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') }} - uses: ./.github/workflows/multi_arch_build_windows_artifacts.yml - secrets: inherit - with: - stage_name: debug-tools - stage_display_name: "Stage - Debug Tools" - timeout_minutes: 180 # 3 hours - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} - build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - build_runs_on: ${{ inputs.build_runs_on }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - external_repo_config: ${{ inputs.external_repo_config }} - permissions: - contents: read - id-token: write diff --git a/.github/workflows/multi_arch_build_windows_artifacts.yml b/.github/workflows/multi_arch_build_windows_artifacts.yml deleted file mode 100644 index 0dcef45e8e..0000000000 --- a/.github/workflows/multi_arch_build_windows_artifacts.yml +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable Workflow: Build Stage Artifacts -# -# This workflow builds TheRock stage artifacts for all stages. -# Handles both generic stages (compiler-runtime, etc.) and -# per-arch stages (math-libs) using conditional logic. -# -# Artifacts flow between stages via S3 using the artifact_manager.py tool. -# -# TODO: When kpack splitting is permanently enabled, change fetch commands to use -# --generic-only instead of --amdgpu-families for efficiency. Subsequent build -# stages only need generic (host) artifacts; device-specific kpacks are not -# needed until final packaging. - -name: Build Stage Artifacts - -on: - workflow_call: - inputs: - stage_name: - type: string - required: true - description: "Stage name (e.g., compiler-runtime, math-libs)" - stage_display_name: - type: string - required: true - description: "Human-readable display name for the job" - timeout_minutes: - type: number - required: true - description: "Job timeout in minutes" - amdgpu_family: - type: string - required: false - default: "" - description: "GPU family for per-arch stages (empty for generic stages)" - dist_amdgpu_families: - type: string - required: true - description: "Semicolon-separated list of all GPU families for dist targets" - build_variant_cmake_preset: - type: string - rocm_package_version: - type: string - required: true - description: "ROCm package version string" - release_type: - description: 'Release type; controls artifact bucket and IAM role' - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - build_runs_on: - description: "Build runner label (selected by configure script with weighted distribution)" - type: string - default: "azure-windows-scale-rocm" - external_repo_config: - description: "JSON config for external repo checkout (generated by detect_external_repo_config.py)" - type: string - default: "" - -permissions: - contents: read - -jobs: - build_stage: - name: ${{ inputs.stage_display_name }} - runs-on: ${{ inputs.build_runs_on }} - timeout-minutes: ${{ inputs.timeout_minutes }} - permissions: - contents: read - id-token: write - defaults: - run: - shell: bash - env: - STAGE_NAME: ${{ inputs.stage_name }} - AMDGPU_FAMILIES: ${{ inputs.amdgpu_family }} - BUILD_DIR: B:\build - CCACHE_CONFIGPATH: ${{ github.workspace }}/.ccache/ccache.conf - CACHE_DIR: ${{ github.workspace }}/.cache - TEATIME_FORCE_INTERACTIVE: 0 - RELEASE_TYPE: ${{ inputs.release_type }} - EXTERNAL_FETCH_SOURCES_ARGS: ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).fetch_sources_args || '' }} - - steps: - # Enable git longpaths before checkout (needed for external repos with long paths) - - name: Enable git longpaths - if: ${{ inputs.external_repo_config != '' }} - run: git config --system core.longpaths true - - - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout external repository - if: ${{ inputs.external_repo_config != '' }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ fromJSON(inputs.external_repo_config).repository }} - ref: ${{ fromJSON(inputs.external_repo_config).ref }} - path: ${{ fromJSON(inputs.external_repo_config).checkout_path }} - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install python deps - run: pip install -r requirements.txt - - - name: Install requirements - run: | - choco source disable -n=chocolatey - choco source add -n=internal -s http://nexus-service.nexus-ns.svc.cluster.local:8081/repository/choco-group/ --priority=1 - choco install --no-progress -y ccache - # ninja pinned due to a bug in the 1.13.0 release: - # https://github.com/ninja-build/ninja/issues/2616 - choco install --no-progress -y ninja --version 1.12.1 - choco install --no-progress -y strawberryperl - echo "$PATH;C:\Strawberry\c\bin" >> $GITHUB_PATH - - - uses: iterative/setup-dvc@6c05772f52d19792ed73aedb6c2bf65ec92adebb # v3.0.0 - with: - version: '3.62.0' - - # After other installs, so MSVC gets priority in the PATH. - - name: Configure MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - - name: Adjust git config - run: | - git config --global core.symlinks true - git config --global core.longpaths true - git config fetch.parallel 10 - - - name: Runner health status - run: | - python build_tools/health_status.py - - - name: Setup ccache - run: | - python ./build_tools/setup_ccache.py \ - --release-type "${{ inputs.release_type }}" \ - --dir "$(dirname $CCACHE_CONFIGPATH)" \ - --local-path "$CACHE_DIR/ccache" \ - --log-dir "${BUILD_DIR}/logs/ccache" - - # Note: no AWS credentials should be needed here since our CI buckets - # all currently have public read access. - - name: Fetch inbound artifacts - run: | - python build_tools/artifact_manager.py fetch \ - --run-id="${{ github.run_id }}" \ - --stage="${STAGE_NAME}" \ - --amdgpu-families="${{ inputs.amdgpu_family }}" \ - --output-dir="${BUILD_DIR}" \ - --bootstrap - - - name: Fetch sources - timeout-minutes: 30 - run: python build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1 ${EXTERNAL_FETCH_SOURCES_ARGS} - - - name: Get stage configuration - id: stage_config - run: | - python build_tools/configure_stage.py \ - --stage=${STAGE_NAME} \ - --amdgpu-families="${{ inputs.amdgpu_family }}" \ - --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ - --gha-output - - - name: Install stage python deps - if: ${{ steps.stage_config.outputs.pip_install_cmd }} - run: pip install ${{ steps.stage_config.outputs.pip_install_cmd }} - - - name: Configure - run: | - cmake -B "${BUILD_DIR}" -S . -GNinja \ - --preset=${{ inputs.build_variant_cmake_preset }} \ - -DTHEROCK_PACKAGE_VERSION=${{ inputs.rocm_package_version }} \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - ${{ inputs.external_repo_config != '' && format('-DTHEROCK_{0}_SOURCE_DIR={1}', fromJSON(inputs.external_repo_config).source_package, fromJSON(inputs.external_repo_config).checkout_path) || '' }} \ - ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).extra_cmake_options || '' }} \ - ${{ steps.stage_config.outputs.cmake_args }} - - - name: Build stage - run: | - cmake --build "${BUILD_DIR}" --target stage-${STAGE_NAME} therock-artifacts -- -k 0 - - - name: Run build tests (Comgr, blocking) - if: ${{ !cancelled() && inputs.stage_name == 'compiler-runtime' }} - env: - TEATIME_FORCE_INTERACTIVE: 1 - run: | - cmake --build "${BUILD_DIR}" --target therock-build-tests -- -k 0 - - - name: Report - if: ${{ !cancelled() }} - run: | - echo "CCache Stats:" - ccache -s -v - echo "Artifacts:" - ls -lh "${BUILD_DIR}"/artifacts/*.tar.xz 2>/dev/null || echo "No artifacts found" - - # TODO(scotttodd): enable once these tests pass - # therock-examples-cpp-sdk-user can't find hipsparselt-config.cmake - # - name: Test Packaging - # run: | - # ctest --test-dir ${BUILD_DIR} --output-on-failure - - - name: Configure AWS credentials for artifact uploads - if: ${{ always() }} - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - # TODO(#3336): Attempt upload `always()`? - # * If some part of the build failed, partial artifacts may be useful for debugging. - # * This script fails if `build_dir / "artifacts"` does not exist. That's useful - # if the build succeeded but did not produce artifacts. That's _not_ useful - # if the workflow failed or was cancelled before the build even started though. - - name: Push stage artifacts - run: | - python build_tools/artifact_manager.py push \ - --run-id ${{ github.run_id }} \ - --stage="${STAGE_NAME}" \ - --build-dir="${BUILD_DIR}" - - - name: Upload stage logs - if: ${{ always() }} - run: | - python build_tools/github_actions/post_stage_upload.py \ - --run-id=${{ github.run_id }} \ - --stage="${STAGE_NAME}" \ - --build-dir="${BUILD_DIR}" \ - --amdgpu-family="${AMDGPU_FAMILIES}" diff --git a/.github/workflows/multi_arch_build_windows_pytorch_wheels.yml b/.github/workflows/multi_arch_build_windows_pytorch_wheels.yml deleted file mode 100644 index 5e4ce31f72..0000000000 --- a/.github/workflows/multi_arch_build_windows_pytorch_wheels.yml +++ /dev/null @@ -1,485 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Reusable multi-arch PyTorch build workflow for Windows. -# -# Built shape: -# - One fat build_prod_wheels.py invocation for one python_version and -# pytorch_git_ref, covering all gfx targets expanded from the -# inputs.amdgpu_families set. Builds torch, torchaudio, and torchvision -# in a single invocation. -# - kpack-split the fat torch and torchvision wheels into host wheels -# plus per-gfx amd-torch-device-* / amd-torchvision-device-* wheels. -# other packages ship as-is (nothing to split). -# - Upload the result to s3://therock-{release_type}-python/v4/whl/ -# via publish_pytorch_to_release_bucket.py (pip-compatible index generation -# happens server-side). -# -# For convergence with CI and testing plans see: -# * https://github.com/ROCm/TheRock/issues/3332 -# * https://github.com/ROCm/TheRock/issues/3291 - -name: Multi-Arch Build Windows PyTorch Wheels - -on: - workflow_call: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx110X-all;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - test_amdgpu_families: - description: >- - Semicolon-separated AMD GPU families to run tests for (e.g. - 'gfx120X-all'). Use 'auto' to test built families, or 'none' to skip - tests. - type: string - default: "auto" - python_version: - description: Python version to build wheels for. - type: string - required: true - pytorch_git_ref: - description: PyTorch ref to checkout. Typically "nightly" or "release/X.Y". - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; selects the S3 bucket.' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type ('none', 'sccache', or 'ccache')" - type: string - default: "sccache" - workflow_dispatch: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx110X-all;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - test_amdgpu_families: - description: >- - Semicolon-separated AMD GPU families to run tests for (e.g. - 'gfx120X-all'). Use 'auto' to test built families, or 'none' to skip - tests. - type: string - default: "auto" - python_version: - description: Python version to build wheels for (for example, "3.12"). - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (for example, "release/2.12"). - type: string - default: "release/2.12" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - -run-name: Build Multi-Arch Windows PyTorch Wheels (${{ inputs.release_type }}, ${{ inputs.python_version }}, ${{ inputs.pytorch_git_ref }}) - -permissions: - contents: read - -jobs: - build_pytorch_wheels: - name: Build | py ${{ inputs.python_version }} | torch ${{ inputs.pytorch_git_ref }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-windows-scale-rocm' || 'windows-2022' }} - permissions: - id-token: write - contents: read - env: - CHECKOUT_ROOT: B:/src - # Note the \ here instead of /. This should be used from 'cmd' not 'bash'! - PACKAGE_DIST_DIR: ${{ github.workspace }}\output\packages\dist - optional_build_prod_arguments: "" - SCCACHE_BUCKET: "therock-pytorch-sccache-${{ inputs.release_type }}" - SCCACHE_REGION: us-east-1 - SCCACHE_S3_KEY_PREFIX: "windows/multiarch/v1/" - SCCACHE_S3_USE_SSL: "true" - SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "true" - SCCACHE_IDLE_TIMEOUT: "0" # never idle-out; keeps stats accurate on long builds - SCCACHE_LOG: warn - outputs: - torch_version: ${{ steps.pytorch-versions.outputs.torch_version }} - torchaudio_version: ${{ steps.pytorch-versions.outputs.torchaudio_version }} - torchvision_version: ${{ steps.pytorch-versions.outputs.torchvision_version }} - package_index_url: ${{ steps.publish-pytorch-wheels.outputs.package_index_url }} - defaults: - run: - # Note: there are mixed uses of 'bash' (this default) and 'cmd' below - shell: bash - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - - name: Select Python version - run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} - - # TODO(amd-justchen): share with build_windows_artifacts.yml. Include in VM image? Dockerfile? - - name: Install requirements - run: | - choco source disable -n=chocolatey - choco source add -n=internal -s http://nexus-service.nexus-ns.svc.cluster.local:8081/repository/choco-group/ --priority=1 - choco install --no-progress -y ninja --version 1.13.1 - - # Build a fixed version of libuv from source, no matter the pytorch ref. - # gloo requires >= 1.26; v1.47.0 matches upstream's source build. - # https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/windows/arm64/bootstrap_libuv.bat - - name: Build libuv from source for GLOO - run: | - git clone https://github.com/libuv/libuv.git -b v1.47.0 --depth 1 - cmake -S libuv -B libuv/build -DBUILD_TESTING=OFF - cmake --build libuv/build --config Release - cmake --install libuv/build --prefix "${{ github.workspace }}/libuv-install" - echo "libuv_ROOT=${{ github.workspace }}/libuv-install" >> $GITHUB_ENV - - - name: Install sccache - if: ${{ inputs.cache_type == 'sccache' }} - run: | - choco install --no-progress -y sccache - echo "sccache version: $(sccache --version)" - - # After other installs, so MSVC gets priority in the PATH. - - name: Configure MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - - name: Install python deps for CI - run: | - pip install -r external-builds/pytorch/requirements-ci.txt - - - name: Checkout PyTorch source repos (nightly) - if: ${{ inputs.pytorch_git_ref == 'nightly' }} - run: | - git config --global core.longpaths true - python ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --repo-hashtag nightly \ - --no-commit-hipify - python ./external-builds/pytorch/pytorch_audio_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch_audio \ - --repo-hashtag nightly \ - --no-commit-hipify - python ./external-builds/pytorch/pytorch_vision_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch_vision \ - --repo-hashtag nightly \ - --no-commit-hipify - - - name: Checkout PyTorch source repos (stable) - if: ${{ inputs.pytorch_git_ref != 'nightly' }} - run: | - git config --global core.longpaths true - python ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --gitrepo-origin https://github.com/ROCm/pytorch.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-commit-hipify - python ./external-builds/pytorch/pytorch_audio_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch_audio \ - --torch-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --require-related-commit \ - --no-commit-hipify - python ./external-builds/pytorch/pytorch_vision_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch_vision \ - --torch-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --require-related-commit \ - --no-commit-hipify - - # determine_version.py sets optional_build_prod_arguments in - # GITHUB_ENV (--rocm-sdk-version, --version-suffix). - - name: Determine optional arguments passed to `build_prod_wheels.py` - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version ${{ inputs.rocm_version }} - - - name: Configure AWS Credentials for sccache - if: ${{ inputs.cache_type == 'sccache' && github.repository_owner == 'ROCm' }} - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::324352301041:role/therock-${{ inputs.release_type }} - # Multi-package PyTorch builds can exceed the default 1h credential - # window; the role's MaxSessionDuration must be >= this value. - role-duration-seconds: 21600 - special-characters-workaround: true - - - name: Verify sccache - if: ${{ inputs.cache_type == 'sccache' }} - run: | - echo "sccache version: $(sccache --version)" - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" - echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}" - - - name: Expand amdgpu families to gfx targets - id: expand - run: | - targets=$(python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_families }}") - echo "Expanded families '${{ inputs.amdgpu_families }}' -> '${targets}'" - echo "targets=${targets}" >> "$GITHUB_OUTPUT" - - # Using 'cmd' here is load bearing! There are configuration issues when - # run under 'bash': https://github.com/ROCm/TheRock/issues/827#issuecomment-3025858800 - - name: Build PyTorch wheels - shell: cmd - run: | - set "CACHE_FLAG=" - if "${{ inputs.cache_type }}"=="sccache" set "CACHE_FLAG=--use-sccache" - if "${{ inputs.cache_type }}"=="ccache" set "CACHE_FLAG=--use-ccache" - - echo "Building PyTorch wheels for ${{ inputs.amdgpu_families }} (cache: ${{ inputs.cache_type }})" - python ./external-builds/pytorch/build_prod_wheels.py ^ - build ^ - --install-rocm ^ - --index-url="${{ inputs.rocm_package_index_url }}" ^ - --find-links="${{ inputs.rocm_package_find_links_url }}" ^ - --root-checkout-dir ${{ env.CHECKOUT_ROOT }} ^ - --enable-pytorch-flash-attention-windows ^ - --clean ^ - --output-dir ${{ env.PACKAGE_DIST_DIR }} ^ - %CACHE_FLAG% ^ - --pytorch-rocm-arch ${{ steps.expand.outputs.targets }} ^ - ${{ env.optional_build_prod_arguments }} - - - name: Report cache stats - if: ${{ !cancelled() && inputs.cache_type != 'none' }} - run: | - if [ "${{ inputs.cache_type }}" = "sccache" ]; then - echo "sccache stats:" - echo "--------------" - sccache --show-stats || true - elif [ "${{ inputs.cache_type }}" = "ccache" ]; then - echo "ccache stats:" - echo "-------------" - ccache -s -v || true - fi - - - name: Sanity check wheel - shell: cmd - run: | - python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }} - - # The kpack wheel splitter takes the fat torch wheel and produces a - # smaller host wheel plus one device wheel per gfx target. Source-install - # from the TheRock checkout - kpack has no published wheel today. - # The outer actions/checkout skips submodules for speed, so pull in - # rocm-systems only when we actually need it. - - name: Init rocm-systems submodule - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --depth=1 rocm-systems - - - name: Install kpack tooling - run: | - pip install \ - -e rocm-systems/python/rocm-bootstrap \ - -e rocm-systems/shared/kpack - - - name: Split fat wheels (torch + torchvision) - run: | - DIST_DIR="$(cygpath -u "${{ env.PACKAGE_DIST_DIR }}")" - - # LLVM binaries ship inside the already-installed rocm-sdk-devel. - ROCM_SDK_ROOT="$(cygpath -u "$(python -m rocm_sdk path --root)")" - LLVM_BIN="${ROCM_SDK_ROOT}/lib/llvm/bin" - if [ ! -d "${LLVM_BIN}" ]; then - echo "ERROR: LLVM bin dir not found: ${LLVM_BIN}" >&2 - exit 1 - fi - - # The kpack Toolchain strictly looks up "objcopy" and "readelf" on - # PATH. On Windows, ROCm ships only llvm-objcopy.exe / - # llvm-readelf.exe, so stage copies under the unprefixed names. - # TODO: drop this shim once rocm-kpack's Toolchain adds llvm-* - # fallbacks for objcopy/readelf (parallels its existing objdump - # pattern). See https://github.com/ROCm/rocm-systems/issues/5506. - SHIMS="$(cygpath -u "${RUNNER_TEMP}")/kpack-shims" - mkdir -p "${SHIMS}" - cp "${LLVM_BIN}/llvm-objcopy.exe" "${SHIMS}/objcopy.exe" - cp "${LLVM_BIN}/llvm-readelf.exe" "${SHIMS}/readelf.exe" - - # Prepend shims + LLVM bin so clang-offload-bundler.exe and the - # llvm-objdump fallback are also resolvable. - export PATH="${SHIMS}:${LLVM_BIN}:${PATH}" - - # The splitter writes the host wheel under the same filename as the - # input, so relocate each fat wheel out of PACKAGE_DIST_DIR before - # invoking the splitter. torchaudio ships as-is. - FAT_STAGE="$(cygpath -u "${RUNNER_TEMP}")/pytorch-fat" - mkdir -p "${FAT_STAGE}" - - ROCM_SDK_VERSION="$(python -m rocm_sdk version)" - - mv "${DIST_DIR}"/torch-*.whl "${FAT_STAGE}/" - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torch-*.whl \ - --output-dir "${DIST_DIR}" \ - --device-package-prefix amd-torch-device \ - --overlay-root torch/ \ - --wheel-type torch-fat \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j 32 - rm -f "${FAT_STAGE}"/torch-*.whl - - mv "${DIST_DIR}"/torchvision-*.whl "${FAT_STAGE}/" - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torchvision-*.whl \ - --output-dir "${DIST_DIR}" \ - --device-package-prefix amd-torchvision-device \ - --overlay-root torchvision/ \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j 32 - - rm -rf "${FAT_STAGE}" - echo "Post-split contents of ${DIST_DIR}:" - ls -lh "${DIST_DIR}/" - - # TODO(#5110): Once manifest-driven builds land, write these outputs from - # the manifest and keep wheel scanning as a validation step that the build - # produced the requested package versions. - - name: Write PyTorch package version outputs - id: pytorch-versions - shell: cmd - run: | - python build_tools/github_actions/write_torch_versions.py --dist-dir ${{ env.PACKAGE_DIST_DIR }} - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Upload PyTorch wheels to release bucket - id: publish-pytorch-wheels - run: | - python build_tools/github_actions/publish_pytorch_to_release_bucket.py \ - --source-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --release-type="${{ inputs.release_type }}" - - configure_pytorch_tests: - name: Configure PyTorch Tests - runs-on: ubuntu-24.04 - permissions: - contents: read - outputs: - enabled: ${{ steps.configure.outputs.enabled }} - matrix: ${{ steps.configure.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - - name: Configure PyTorch test matrix - id: configure - env: - CI_CONFIG_PATH: ci-config - run: | - python build_tools/github_actions/configure_pytorch_test_matrix.py \ - --build-amdgpu-families "${{ inputs.amdgpu_families }}" \ - --test-amdgpu-families "${{ inputs.test_amdgpu_families }}" \ - --platform windows - - test_pytorch_wheels: - name: Test | ${{ matrix.amdgpu_family }} | ${{ matrix.test_runs_on }} - if: ${{ needs.configure_pytorch_tests.outputs.enabled == 'true' }} - needs: [build_pytorch_wheels, configure_pytorch_tests] - permissions: - contents: read - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.configure_pytorch_tests.outputs.matrix) }} - uses: ./.github/workflows/test_pytorch_wheels.yml - with: - amdgpu_family: ${{ matrix.amdgpu_family }} - test_runs_on: ${{ matrix.test_runs_on }} - package_index_url: ${{ needs.build_pytorch_wheels.outputs.package_index_url }} - python_version: ${{ inputs.python_version }} - torch_version: ${{ needs.build_pytorch_wheels.outputs.torch_version }} - # TODO(#5110): pass manifest_url here and let test_pytorch_wheels.yml use - # checkout_from_manifest.py --no-hipify --no-submodules so tests use the - # exact source commit that produced the package under test. - pytorch_git_ref: ${{ inputs.pytorch_git_ref }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} diff --git a/.github/workflows/multi_arch_build_windows_pytorch_wheels_ci.yml b/.github/workflows/multi_arch_build_windows_pytorch_wheels_ci.yml deleted file mode 100644 index 451c5e0a93..0000000000 --- a/.github/workflows/multi_arch_build_windows_pytorch_wheels_ci.yml +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# CI variant of multi_arch_build_windows_pytorch_wheels.yml. -# -# Key differences from the release workflow: -# - Installs ROCm packages from either a flat find-links page (CI artifacts, -# via rocm_package_find_links_url) or a PEP 503 index (via -# rocm_package_index_url); provide whichever one applies. -# - Builds torch only (no torchvision, torchaudio) -# - No S3 upload, staging, testing, or promotion — just build + sanity check -# -# TODO(#3291): Build more packages (torchvision, torchaudio, etc.) -# -# Both workflows share build_prod_wheels.py for the actual build logic. -# See https://github.com/ROCm/TheRock/issues/3291 for convergence plans. - -name: Build Windows PyTorch Wheels (CI) - -on: - workflow_call: - inputs: - artifact_group: - type: string - required: true - python_version: - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (typically "release/X.Y") - type: string - default: "release/2.11" - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - cache_type: - description: "Compiler cache type (sccache, ccache, or none)" - required: false - type: string - default: "sccache" - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx110X-all;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. Leave empty - for legacy callers that rely on build_prod_wheels.py's - `rocm-sdk targets` fallback. - type: string - default: "" - kpack_split: - description: "'true' if this is a kpack-split flat build." - type: string - default: "false" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - workflow_dispatch: - inputs: - artifact_group: - description: "The artifact group to build (e.g. gfx110X-all, gfx120X-all)" - type: string - default: gfx1151 - python_version: - type: string - default: "3.12" - pytorch_git_ref: - description: PyTorch ref to checkout (typically "release/X.Y") - type: string - default: "release/2.11" - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - -permissions: - contents: read - -run-name: Build Windows PyTorch Wheels CI (${{ inputs.artifact_group }}, py${{ inputs.python_version }}, ${{ inputs.pytorch_git_ref }}) - -jobs: - build_pytorch_wheels: - name: Build PyTorch | ${{ inputs.artifact_group }} | torch ${{ inputs.pytorch_git_ref }} | py${{ inputs.python_version }} - runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-windows-scale-rocm' || 'windows-2022' }} - permissions: - id-token: write - contents: read - env: - CHECKOUT_ROOT: B:/src - # Note the \ here instead of /. This should be used from 'cmd' not 'bash'! - PACKAGE_DIST_DIR: ${{ github.workspace }}\output\packages\dist - RELEASE_TYPE: ci - optional_build_prod_arguments: "" - SCCACHE_BUCKET: therock-pytorch-sccache-ci - SCCACHE_REGION: us-east-1 - SCCACHE_S3_KEY_PREFIX: "windows/${{ inputs.artifact_group }}/" - SCCACHE_S3_USE_SSL: "true" - SCCACHE_S3_SERVER_SIDE_ENCRYPTION: "true" - SCCACHE_IDLE_TIMEOUT: "0" # never idle-out; keeps stats accurate on long builds - SCCACHE_LOG: warn - defaults: - run: - # Note: there are mixed uses of 'bash' (this default) and 'cmd' below - shell: bash - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Configure Git Identity - run: | - git config --global user.name "therockbot" - git config --global user.email "therockbot@amd.com" - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - - name: Select Python version - run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} - - # sccache uses an S3 bucket reached via an OIDC-assumed IAM role, which - # fork PRs / external repos cannot assume. Compute the effective cache - # type once (downgrading sccache -> none on forks) so every sccache step - # below keys off a single decision instead of repeating fork checks. - - name: Determine cache type - id: cache - run: | - python build_tools/github_actions/compute_pytorch_cache_type.py \ - --cache-type "${{ inputs.cache_type }}" - - # TODO(amd-justchen): share with build_windows_artifacts.yml. Include in VM image? Dockerfile? - - name: Install requirements - run: | - choco source disable -n=chocolatey - choco source add -n=internal -s http://nexus-service.nexus-ns.svc.cluster.local:8081/repository/choco-group/ --priority=1 - choco install --no-progress -y ninja --version 1.13.1 - - # Build a fixed version of libuv from source, no matter the pytorch ref. - # gloo requires >= 1.26; v1.47.0 matches upstream's source build. - # https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/windows/arm64/bootstrap_libuv.bat - - name: Build libuv from source for GLOO - run: | - git clone https://github.com/libuv/libuv.git -b v1.47.0 --depth 1 - cmake -S libuv -B libuv/build -DBUILD_TESTING=OFF - cmake --build libuv/build --config Release - cmake --install libuv/build --prefix "${{ github.workspace }}/libuv-install" - echo "libuv_ROOT=${{ github.workspace }}/libuv-install" >> $GITHUB_ENV - - - name: Install sccache - if: ${{ steps.cache.outputs.cache_type == 'sccache' }} - run: | - choco install --no-progress -y sccache - echo "sccache version: $(sccache --version)" - - # After other installs, so MSVC gets priority in the PATH. - - name: Configure MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - - name: Install python deps for CI - run: | - pip install -r external-builds/pytorch/requirements-ci.txt - - - name: Checkout PyTorch source (nightly) - if: ${{ inputs.pytorch_git_ref == 'nightly' }} - run: | - git config --global core.longpaths true - python ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --repo-hashtag nightly \ - --no-commit-hipify - - - name: Checkout PyTorch source (stable) - if: ${{ inputs.pytorch_git_ref != 'nightly' }} - run: | - git config --global core.longpaths true - python ./external-builds/pytorch/pytorch_torch_repo.py checkout \ - --checkout-dir ${{ env.CHECKOUT_ROOT }}/pytorch \ - --gitrepo-origin https://github.com/ROCm/pytorch.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-commit-hipify - - # Note: determine_version.py sets optional_build_prod_arguments in - # GITHUB_ENV, which includes --rocm-sdk-version and --version-suffix. - - name: Determine optional arguments passed to `build_prod_wheels.py` - if: ${{ inputs.rocm_version }} - run: | - python build_tools/github_actions/determine_version.py \ - --rocm-version ${{ inputs.rocm_version }} - - - name: Configure AWS Credentials for sccache - if: ${{ steps.cache.outputs.cache_type == 'sccache' }} - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::324352301041:role/therock-ci - # Multi-package PyTorch builds can exceed the default 1h credential - # window; the role's MaxSessionDuration must be >= this value. - role-duration-seconds: 21600 - special-characters-workaround: true - - - name: Verify sccache - if: ${{ steps.cache.outputs.cache_type == 'sccache' }} - run: | - echo "sccache version: $(sccache --version)" - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" - echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}" - - - name: Expand amdgpu families to gfx targets - if: ${{ inputs.amdgpu_families != '' }} - id: expand - run: | - targets=$(python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_families }}") - echo "Expanded families '${{ inputs.amdgpu_families }}' -> '${targets}'" - echo "targets=${targets}" >> "$GITHUB_OUTPUT" - - # Using 'cmd' here is load bearing! There are configuration issues when - # run under 'bash': https://github.com/ROCm/TheRock/issues/827#issuecomment-3025858800 - # - # inputs.amdgpu_families is only populated by the multi-arch caller - # today. Legacy single-family CI (ci_linux.yml) leaves it empty and - # falls back to `rocm-sdk targets` inside build_prod_wheels.py. - # TODO(#4687): Make --pytorch-rocm-arch required here once every - # caller plumbs the value through and the fallback in - # build_prod_wheels.py::get_rocm_sdk_targets is removed. - - name: Build PyTorch wheels - shell: cmd - run: | - set "CACHE_FLAG=" - if "${{ steps.cache.outputs.cache_type }}"=="sccache" set "CACHE_FLAG=--use-sccache" - if "${{ steps.cache.outputs.cache_type }}"=="ccache" set "CACHE_FLAG=--use-ccache" - - set "PYTORCH_ROCM_ARCH_FLAG=" - if not "${{ steps.expand.outputs.targets }}"=="" set "PYTORCH_ROCM_ARCH_FLAG=--pytorch-rocm-arch ${{ steps.expand.outputs.targets }}" - - echo "Building PyTorch wheels for ${{ inputs.artifact_group }} (cache: ${{ steps.cache.outputs.cache_type }})" - python ./external-builds/pytorch/build_prod_wheels.py ^ - build ^ - --install-rocm ^ - --index-url="${{ inputs.rocm_package_index_url }}" ^ - --find-links="${{ inputs.rocm_package_find_links_url }}" ^ - --root-checkout-dir ${{ env.CHECKOUT_ROOT }} ^ - --enable-pytorch-flash-attention-windows ^ - --clean ^ - --output-dir ${{ env.PACKAGE_DIST_DIR }} ^ - %CACHE_FLAG% ^ - %PYTORCH_ROCM_ARCH_FLAG% ^ - ${{ env.optional_build_prod_arguments }} - - - name: Report cache stats - if: ${{ !cancelled() && steps.cache.outputs.cache_type != 'none' }} - run: | - if [ "${{ steps.cache.outputs.cache_type }}" = "sccache" ]; then - echo "sccache stats:" - echo "--------------" - sccache --show-stats || true - elif [ "${{ steps.cache.outputs.cache_type }}" = "ccache" ]; then - echo "ccache stats:" - echo "-------------" - ccache -s -v || true - fi - - - name: Sanity check wheel - shell: cmd - run: | - python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }} - - # The kpack wheel splitter takes the fat torch wheel and produces a - # smaller host wheel plus one device wheel per gfx target. Source-install - # from the TheRock checkout — kpack has no published wheel today. - # The outer actions/checkout skips submodules for speed, so pull in - # rocm-systems only when we actually need it. - - name: Init rocm-systems submodule - if: ${{ inputs.kpack_split == 'true' }} - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --depth=1 rocm-systems - - - name: Install kpack tooling - if: ${{ inputs.kpack_split == 'true' }} - run: | - pip install \ - -e rocm-systems/python/rocm-bootstrap \ - -e rocm-systems/shared/kpack - - - name: Split PyTorch fat wheel - if: ${{ inputs.kpack_split == 'true' }} - run: | - DIST_DIR="$(cygpath -u "${{ env.PACKAGE_DIST_DIR }}")" - - # LLVM binaries ship inside the already-installed rocm-sdk-devel. - ROCM_SDK_ROOT="$(cygpath -u "$(python -m rocm_sdk path --root)")" - LLVM_BIN="${ROCM_SDK_ROOT}/lib/llvm/bin" - if [ ! -d "${LLVM_BIN}" ]; then - echo "ERROR: LLVM bin dir not found: ${LLVM_BIN}" >&2 - exit 1 - fi - - # The kpack Toolchain strictly looks up "objcopy" and "readelf" on - # PATH. On Windows, ROCm ships only llvm-objcopy.exe / - # llvm-readelf.exe, so stage copies under the unprefixed names. - # TODO: drop this shim once rocm-kpack's Toolchain adds llvm-* - # fallbacks for objcopy/readelf (parallels its existing objdump - # pattern). See https://github.com/ROCm/rocm-systems/issues/5506. - SHIMS="$(cygpath -u "${RUNNER_TEMP}")/kpack-shims" - mkdir -p "${SHIMS}" - cp "${LLVM_BIN}/llvm-objcopy.exe" "${SHIMS}/objcopy.exe" - cp "${LLVM_BIN}/llvm-readelf.exe" "${SHIMS}/readelf.exe" - - # Prepend shims + LLVM bin so clang-offload-bundler.exe and the - # llvm-objdump fallback are also resolvable. - export PATH="${SHIMS}:${LLVM_BIN}:${PATH}" - - # The splitter writes the host wheel under the same filename as the - # input, so relocate the fat wheel out of PACKAGE_DIST_DIR first. - FAT_STAGE="$(cygpath -u "${RUNNER_TEMP}")/pytorch-fat" - mkdir -p "${FAT_STAGE}" - mv "${DIST_DIR}"/torch-*.whl "${FAT_STAGE}/" - - ROCM_SDK_VERSION="$(python -m rocm_sdk version)" - - python -m rocm_kpack.tools.split_python_wheels \ - --input "${FAT_STAGE}"/torch-*.whl \ - --output-dir "${DIST_DIR}" \ - --device-package-prefix amd-torch-device \ - --overlay-root torch/ \ - --wheel-type torch-fat \ - --device-requires-dist "rocm-sdk-device-@GFXARCH@ == ${ROCM_SDK_VERSION}" \ - --compression zstd \ - -j 32 - - rm -rf "${FAT_STAGE}" - echo "Post-split contents of ${DIST_DIR}:" - ls -lh "${DIST_DIR}/" - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ci - - # Appends to the same therock-ci-artifacts/-windows/python/ prefix - # that the ROCm python job already populated — one find-links URL - # covers the whole SDK + PyTorch stack. - - name: Upload PyTorch wheels - run: | - python build_tools/github_actions/upload_python_packages.py \ - --input-packages-dir="${{ env.PACKAGE_DIST_DIR }}" \ - --artifact-group="${{ inputs.artifact_group }}" \ - --run-id="${{ github.run_id }}" \ - --multiarch diff --git a/.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml b/.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml deleted file mode 100644 index 290d671a9f..0000000000 --- a/.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Build WSL ROCDXG Artifacts - -on: - workflow_call: - inputs: - stage_name: - type: string - required: true - description: "Stage name" - stage_display_name: - type: string - required: true - description: "Human-readable display name for the job" - timeout_minutes: - type: number - required: true - description: "Job timeout in minutes" - dist_amdgpu_families: - type: string - required: true - description: "Semicolon-separated list of all GPU families for dist targets" - rocm_package_version: - type: string - required: true - description: "ROCm package version string" - build_variant_cmake_preset: - type: string - required: false - default: "" - description: "CMake preset name for this build variant (optional)" - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - external_repo_config: - description: "JSON config for external repo checkout (generated by detect_external_repo_config.py)" - type: string - default: "" - -permissions: - contents: read - -jobs: - build_stage: - name: ${{ inputs.stage_display_name }} - runs-on: - group: ROCR-Runtime-libhsakmt-WSL - timeout-minutes: ${{ inputs.timeout_minutes }} - permissions: - contents: read - id-token: write - defaults: - run: - shell: bash - env: - STAGE_NAME: ${{ inputs.stage_name }} - RELEASE_TYPE: ${{ inputs.release_type }} - THEROCK_HOST_WORKSPACE: ${{ github.workspace }} - EXTERNAL_FETCH_SOURCES_ARGS: ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).fetch_sources_args || '' }} - MANYLINUX_IMAGE: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:a382085df3ba2419b58aa9051350883a0d0b732a4bc0a4ef60458f8161bb08c6 - # Clean, space-free mount target for the Windows SDK shared include dir - # inside the container. Avoids the raw "(x86)" path segment that /bin/sh - # misparses during nested subproject configure. - THEROCK_WSL_WIN_SDK_MOUNT: /opt/winsdk-shared - # Injects Windows env vars into WSL - WSLENV: THEROCK_HOST_WORKSPACE/p:GITHUB_REPOSITORY:GITHUB_EVENT_NAME:GITHUB_EVENT_PATH/p:RELEASE_TYPE:MANYLINUX_IMAGE:THEROCK_WSL_WIN_SDK_MOUNT:THEROCK_WIN_SDK_SHARED/p - - steps: - # Enable git longpaths before checkout (needed for external repos with long paths) - - name: Enable git longpaths - if: ${{ inputs.external_repo_config != '' }} - run: git config --system core.longpaths true - - - name: Checkout Repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout external repository - if: ${{ inputs.external_repo_config != '' }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ fromJSON(inputs.external_repo_config).repository }} - ref: ${{ fromJSON(inputs.external_repo_config).ref }} - path: ${{ fromJSON(inputs.external_repo_config).checkout_path }} - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install python deps - run: pip install -r requirements.txt - - - name: Runner health status - run: python build_tools/health_status.py - - - name: Adjust git config - run: | - git config --global core.symlinks true - git config --global core.longpaths true - git config fetch.parallel 10 - - - name: Configure MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - - name: Detect Windows SDK version - id: detect_sdk - shell: powershell - env: - WINDOWS_SDK_INCLUDE_ROOT: C:\Program Files (x86)\Windows Kits\10\Include - run: | - if (-not (Test-Path $env:WINDOWS_SDK_INCLUDE_ROOT)) { - throw "Windows SDK not found at $env:WINDOWS_SDK_INCLUDE_ROOT" - } - $versions = Get-ChildItem $env:WINDOWS_SDK_INCLUDE_ROOT -Directory | Sort-Object Name -Descending - if ($versions.Count -eq 0) { - throw "No Windows SDK versions found under $env:WINDOWS_SDK_INCLUDE_ROOT" - } - $latestVersion = $versions[0].Name - "sdk_version=$latestVersion" >> $env:GITHUB_OUTPUT - - - uses: iterative/setup-dvc@6c05772f52d19792ed73aedb6c2bf65ec92adebb # v3.0.0 - with: - version: "3.62.0" - - - name: Fetch sources - timeout-minutes: 30 - run: python build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1 ${EXTERNAL_FETCH_SOURCES_ARGS} - - - name: Set up WSL with Ubuntu - uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 - with: - distribution: Ubuntu-24.04 - update: true - additional-packages: docker.io python3 python3-pip python3-psutil python3-venv rsync - - - name: Start Docker in WSL - shell: wsl-bash {0} - run: | - # WSL2 has no init system here, so launch the daemon detached and wait - # for the socket before any docker run. - nohup dockerd >/tmp/dockerd.log 2>&1 & - for _ in $(seq 1 30); do - if docker info >/dev/null 2>&1; then - break - fi - sleep 1 - done - if ! docker info >/dev/null 2>&1; then - echo "dockerd failed to start" - cat /tmp/dockerd.log - exit 1 - fi - docker pull "${MANYLINUX_IMAGE}" - - - name: Set up WSL venv - shell: wsl-bash {0} - run: | - # setup-wsl's default wsl-bash wrapper runs `bash --noprofile --norc -euo pipefail`. - # Explicitly bridged path vars from the Windows host are available in wsl-bash via - # WSLENV. Keep this path WSL-local. - wsl_work_dir="${HOME}/therock-wsl-rocdxg" - wsl_venv_dir="${wsl_work_dir}/.venv" - python3 "${THEROCK_HOST_WORKSPACE}/build_tools/setup_venv.py" --clean "${wsl_venv_dir}" - "${wsl_venv_dir}/bin/python" -m pip install -r "${THEROCK_HOST_WORKSPACE}/requirements.txt" - - - name: Get stage configuration - id: stage_config - # This is a pure topology calculation, so keep it on the Windows host - # where GitHub step outputs are native. The explicit --platform=linux - # still makes configure_stage.py emit Linux/WSL feature selections. - run: | - python build_tools/configure_stage.py \ - --stage="${STAGE_NAME}" \ - --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ - --platform=linux \ - --gha-output - - - name: Install stage python deps - if: ${{ steps.stage_config.outputs.pip_install_cmd }} - shell: wsl-bash {0} - run: | - wsl_venv_dir="${HOME}/therock-wsl-rocdxg/.venv" - "${wsl_venv_dir}/bin/python" -m pip install ${{ steps.stage_config.outputs.pip_install_cmd }} - - - name: Prepare WSL paths - shell: wsl-bash {0} - env: - # Made available in WSL via job-level WSLENV - THEROCK_WIN_SDK_SHARED: C:\Program Files (x86)\Windows Kits\10\Include\${{ steps.detect_sdk.outputs.sdk_version }}\shared - run: | - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - if [ ! -d "${THEROCK_WIN_SDK_SHARED}" ]; then - echo "Windows SDK include path not found: ${THEROCK_WIN_SDK_SHARED}" - exit 1 - fi - mkdir -p "${wsl_build_dir}" - - - name: Sync source tree to WSL - shell: wsl-bash {0} - run: | - # Subproject source files are checked out on the Windows side via actions/checkout. - # TheRock runs `git diff` against every subproject to check for changed files. - # File operations between Windows and WSL filesystems have a lot of overhead, - # so this method of diffing subprojects becomes very time consuming. - # To resolve this, copy the entire tree in one sequential operation from Windows to WSL. - # All `git diff` are now run on WSL-native files, giving a big speed boost. - wsl_src_dir="${HOME}/therock-wsl-rocdxg/src" - rm -rf "${wsl_src_dir}" - mkdir -p "${wsl_src_dir}" - rsync -a "${THEROCK_HOST_WORKSPACE}/" "${wsl_src_dir}/" - - - name: Fetch inbound artifacts - shell: wsl-bash {0} - run: | - stage_name="${{ inputs.stage_name }}" - wsl_venv_dir="${HOME}/therock-wsl-rocdxg/.venv" - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - mkdir -p "${wsl_build_dir}" - "${wsl_venv_dir}/bin/python" "${THEROCK_HOST_WORKSPACE}/build_tools/artifact_manager.py" fetch \ - --run-id="${{ github.run_id }}" \ - --run-github-repo="${{ github.repository }}" \ - --platform=linux \ - --stage="${stage_name}" \ - --output-dir="${wsl_build_dir}" \ - --bootstrap - - - name: Configure - shell: wsl-bash {0} - env: - # Made available in WSL via job-level WSLENV - THEROCK_WIN_SDK_SHARED: C:\Program Files (x86)\Windows Kits\10\Include\${{ steps.detect_sdk.outputs.sdk_version }}\shared - run: | - wsl_src_dir="${HOME}/therock-wsl-rocdxg/src" - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - docker run --rm \ - --mount "type=bind,src=${wsl_src_dir},dst=${wsl_src_dir}" \ - --mount "type=bind,src=${wsl_build_dir},dst=${wsl_build_dir}" \ - --mount "type=bind,src=${THEROCK_WIN_SDK_SHARED},dst=${THEROCK_WSL_WIN_SDK_MOUNT},readonly" \ - -w "${wsl_src_dir}" \ - -e "WSL_BUILD_DIR=${wsl_build_dir}" \ - "${MANYLINUX_IMAGE}" \ - bash -c ' - set -euo pipefail - git config --global --add safe.directory "*" - pip install -r requirements.txt - ${{ steps.stage_config.outputs.pip_install_cmd && format('pip install {0}', steps.stage_config.outputs.pip_install_cmd) || 'true' }} - cmake -B "${WSL_BUILD_DIR}" -S . -G Ninja \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DTHEROCK_PACKAGE_VERSION=${{ inputs.rocm_package_version }} \ - -DTHEROCK_WSL_WIN_SDK="'"${THEROCK_WSL_WIN_SDK_MOUNT}"'" \ - ${{ inputs.build_variant_cmake_preset && format('--preset={0}', inputs.build_variant_cmake_preset) }} \ - ${{ inputs.external_repo_config != '' && format('-DTHEROCK_{0}_SOURCE_DIR={1}', fromJSON(inputs.external_repo_config).source_package, fromJSON(inputs.external_repo_config).checkout_path) || '' }} \ - ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).extra_cmake_options || '' }} \ - ${{ steps.stage_config.outputs.cmake_args }} - ' - - - name: Build stage - shell: wsl-bash {0} - env: - # Made available in WSL via job-level WSLENV - THEROCK_WIN_SDK_SHARED: C:\Program Files (x86)\Windows Kits\10\Include\${{ steps.detect_sdk.outputs.sdk_version }}\shared - run: | - stage_name="${{ inputs.stage_name }}" - wsl_src_dir="${HOME}/therock-wsl-rocdxg/src" - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - docker run --rm \ - --mount "type=bind,src=${wsl_src_dir},dst=${wsl_src_dir}" \ - --mount "type=bind,src=${wsl_build_dir},dst=${wsl_build_dir}" \ - --mount "type=bind,src=${THEROCK_WIN_SDK_SHARED},dst=${THEROCK_WSL_WIN_SDK_MOUNT},readonly" \ - -w "${wsl_src_dir}" \ - -e "STAGE_NAME=${stage_name}" \ - -e "WSL_BUILD_DIR=${wsl_build_dir}" \ - "${MANYLINUX_IMAGE}" \ - bash -c ' - set -euo pipefail - cmake --build "${WSL_BUILD_DIR}" --target "stage-${STAGE_NAME}" -- -k 0 - ' - - - name: Report - if: ${{ !cancelled() }} - shell: wsl-bash {0} - run: | - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - echo "Artifacts:" - ls -lh "${wsl_build_dir}"/artifacts/*.tar.xz 2>/dev/null || echo "No artifacts found" - - - name: Configure AWS credentials for artifact uploads - if: ${{ always() && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Push stage artifacts - if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} - shell: wsl-bash {0} - env: - # Append AWS creds set by the "Configure AWS credentials" step - WSLENV: ${{ env.WSLENV }}:AWS_ACCESS_KEY_ID:AWS_SECRET_ACCESS_KEY:AWS_SESSION_TOKEN:AWS_DEFAULT_REGION:AWS_REGION - run: | - stage_name="${{ inputs.stage_name }}" - wsl_venv_dir="${HOME}/therock-wsl-rocdxg/.venv" - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - "${wsl_venv_dir}/bin/python" "${THEROCK_HOST_WORKSPACE}/build_tools/artifact_manager.py" push --run-id ${{ github.run_id }} \ - --run-github-repo="${{ github.repository }}" \ - --platform linux \ - --stage="${stage_name}" \ - --build-dir="${wsl_build_dir}" - - - name: Upload stage logs - if: ${{ always() && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} - shell: wsl-bash {0} - env: - # Append AWS creds set by the "Configure AWS credentials" step - WSLENV: ${{ env.WSLENV }}:AWS_ACCESS_KEY_ID:AWS_SECRET_ACCESS_KEY:AWS_SESSION_TOKEN:AWS_DEFAULT_REGION:AWS_REGION - run: | - wsl_venv_dir="${HOME}/therock-wsl-rocdxg/.venv" - wsl_build_dir="${HOME}/therock-wsl-rocdxg/build" - "${wsl_venv_dir}/bin/python" "${THEROCK_HOST_WORKSPACE}/build_tools/github_actions/post_stage_upload.py" \ - --run-id=${{ github.run_id }} \ - --platform=linux \ - --stage="${{ inputs.stage_name }}" \ - --build-dir="${wsl_build_dir}" \ - --amdgpu-family="" diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml deleted file mode 100644 index e014aa0b0b..0000000000 --- a/.github/workflows/multi_arch_ci.yml +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-Arch CI -# -# This is a staging workflow for the sharded multi-arch build pipeline. -# It mirrors ci.yml but uses multi_arch_build_portable_linux.yml instead of -# ci_linux.yml. Once validated, ci.yml will be updated to use the multi-arch -# sub-workflows directly. - -name: Multi-Arch CI - -on: - push: - branches: - - main - - multi_arch/** - - release/therock-* - workflow_dispatch: - inputs: - linux_amdgpu_families: - type: string - description: "Insert comma-separated list of Linux GPU families to build and test. ex: gfx94X, gfx120X (empty = skip linux)" - default: "" - linux_test_labels: - type: string - description: "Comma-separated test options. Use test:* to filter tests (e.g. test:rocprim). Use test_filter:* to set test type (e.g. test_filter:quick)." - default: "" - windows_amdgpu_families: - type: string - description: "Insert comma-separated list of Windows GPU families to build and test. ex: gfx94X, gfx120X (empty = skip windows)" - default: "" - windows_test_labels: - type: string - description: "Comma-separated test options. Use test:* to filter tests (e.g. test:rocprim). Use test_filter:* to set test type (e.g. test_filter:quick)." - default: "" - prebuilt_stages: - type: string - default: "" - description: "Comma-separated build stages to skip (or 'all' for all stages); artifacts are copied from baseline_run_id instead" - baseline_run_id: - type: string - default: "" - description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set" - stage_reuse_mode: - type: choice - default: "dry-run" - options: - - "dry-run" - - "reuse-stage" - # This drives the AUTOMATIC stage-reuse detection layer, which is - # separate from the manual `prebuilt_stages` input above. dry-run only - # reports which unaffected stages could be reused; reuse-stage actually - # reuses them (skips the build and copies artifacts from a baseline run). - description: "dry-run (report only) | reuse-stage (auto-reuse unaffected stages)" - stage_reuse_max_age_hours: - type: string - default: "72" - description: "Automatic stage reuse: reject baseline runs older than this many hours" - stage_reuse_commit_history: - type: string - default: "50" - description: "Automatic stage reuse: number of recent branch commits to fetch for ancestry" - changed_projects: - type: string - default: "" - description: "Comma-separated list of changed projects to filter tests (empty = run all tests)" - build_pytorch: - type: boolean - default: true - description: "Build PyTorch wheels" - build_jax: - type: boolean - default: true - description: "Build JAX wheels" - pull_request: - types: - - labeled - - opened - - synchronize - -permissions: - contents: read - -concurrency: - # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency - # - # Only allow one run at a time for 'pull_request' events. - # Allow multiple runs for 'push' and 'workflow_dispatch'. - # - # `github.workflow`: each workflow gets its own concurrency group - # `github.head_ref`: the branch name (only defined for pull_request events) - # `github.run_id` : fallback, unique for each run - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - setup: - uses: ./.github/workflows/setup_multi_arch.yml - with: - build_variant: "release" - linux_amdgpu_families: ${{ inputs.linux_amdgpu_families || '' }} - windows_amdgpu_families: ${{ inputs.windows_amdgpu_families || '' }} - linux_test_labels: ${{ inputs.linux_test_labels || '' }} - windows_test_labels: ${{ inputs.windows_test_labels || '' }} - prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} - baseline_run_id: ${{ inputs.baseline_run_id || '' }} - stage_reuse_mode: ${{ inputs.stage_reuse_mode || 'dry-run' }} - stage_reuse_max_age_hours: ${{ inputs.stage_reuse_max_age_hours || '72' }} - stage_reuse_commit_history: ${{ inputs.stage_reuse_commit_history || '50' }} - build_pytorch: ${{ github.event_name != 'workflow_dispatch' || inputs.build_pytorch }} - build_jax: ${{ github.event_name != 'workflow_dispatch' || inputs.build_jax }} - - linux_build_and_test: - name: Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: >- - ${{ - needs.setup.outputs.linux_build_config != '' && - needs.setup.outputs.enable_build_jobs == 'true' - }} - uses: ./.github/workflows/multi_arch_ci_linux.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.linux_build_config }} - test_labels: ${{ needs.setup.outputs.linux_test_labels }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - rocm_deb_package_version: ${{ needs.setup.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - changed_projects: ${{ inputs.changed_projects || '' }} - permissions: - contents: read - id-token: write - - windows_build_and_test: - name: Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: >- - ${{ - needs.setup.outputs.windows_build_config != '' && - needs.setup.outputs.enable_build_jobs == 'true' - }} - uses: ./.github/workflows/multi_arch_ci_windows.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.windows_build_config }} - test_labels: ${{ needs.setup.outputs.windows_test_labels }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - changed_projects: ${{ inputs.changed_projects || '' }} - permissions: - contents: read - id-token: write - - manifest_diff: - name: Manifest Diff - # Refs are auto-derived from push / pull_request events only. - # For manual runs, dispatch manifest-diff.yml directly with explicit refs. - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} - uses: ./.github/workflows/manifest-diff.yml - permissions: - contents: read - id-token: write - - ci_summary: - name: CI Summary - if: always() - needs: - - setup - - linux_build_and_test - - windows_build_and_test - - manifest_diff - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Evaluate workflow results - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python build_tools/github_actions/workflow_summary.py \ - --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/multi_arch_ci_asan.yml b/.github/workflows/multi_arch_ci_asan.yml deleted file mode 100644 index aa07a46b87..0000000000 --- a/.github/workflows/multi_arch_ci_asan.yml +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Multi-Arch CI ASAN - -on: - # TODO: Enable postsubmit ASAN once we have capacity - # push: - # branches: - # - main - workflow_dispatch: - inputs: - linux_amdgpu_families: - type: string - description: "Insert comma-separated list of Linux GPU families to build and test. ex: gfx94X, gfx120X" - default: "" - linux_test_labels: - type: string - description: "If enabled, reduce test set on Linux to the list of labels prefixed with 'test:'. ex: test:rocprim, test:hipcub" - default: "" - prebuilt_stages: - type: string - default: "" - description: "Comma-separated build stages to skip (or 'all' for all stages); artifacts are copied from baseline_run_id instead" - baseline_run_id: - type: string - default: "" - description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set" - pull_request: - types: - - labeled - - opened - - synchronize - -permissions: - contents: read - -concurrency: - # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency - # - # Only allow one run at a time for 'pull_request' events. - # Allow multiple runs for 'push' and 'workflow_dispatch'. - # - # `github.workflow`: each workflow gets its own concurrency group - # `github.head_ref`: the branch name (only defined for pull_request events) - # `github.run_id` : fallback, unique for each run - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - setup: - uses: ./.github/workflows/setup_multi_arch.yml - with: - build_variant: "asan" - linux_amdgpu_families: ${{ inputs.linux_amdgpu_families || '' }} - linux_test_labels: ${{ inputs.linux_test_labels || '' }} - prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} - baseline_run_id: ${{ inputs.baseline_run_id || '' }} - build_pytorch: false - build_jax: false - - linux_build_and_test: - name: Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: >- - ${{ - needs.setup.outputs.linux_build_config != '' && - needs.setup.outputs.enable_build_jobs == 'true' - }} - uses: ./.github/workflows/multi_arch_ci_linux.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.linux_build_config }} - test_labels: ${{ needs.setup.outputs.linux_test_labels }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - rocm_deb_package_version: ${{ needs.setup.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - permissions: - contents: read - id-token: write - - ci_summary: - name: CI Summary - if: always() - needs: - - setup - - linux_build_and_test - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Evaluate workflow results - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python build_tools/github_actions/workflow_summary.py \ - --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/multi_arch_ci_linux.yml b/.github/workflows/multi_arch_ci_linux.yml deleted file mode 100644 index 24db48689d..0000000000 --- a/.github/workflows/multi_arch_ci_linux.yml +++ /dev/null @@ -1,370 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Multi-Arch CI - Linux - -on: - workflow_call: - inputs: - build_config: - type: string - description: >- - JSON object with build configuration for this platform. Fields: - artifact_group, per_family_info, dist_amdgpu_families, - build_variant_label, build_variant_cmake_preset, - build_variant_suffix, build_pytorch, build_jax, - build_native_linux, test_python_packages_matrix, - pytorch_build_matrix, jax_build_matrix, - prebuilt_stages, baseline_run_id. - test_labels: - type: string - rocm_package_version: - type: string - rocm_deb_package_version: - type: string - description: ROCm DEB package version for native packages - rocm_rpm_package_version: - type: string - description: ROCm RPM package version for native packages - test_type: - type: string - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease"' - type: string - default: ci - external_repo_config: - description: "JSON config for external repo checkout (repository, ref, checkout_path, source_package, fetch_sources_args)" - type: string - default: "" - changed_projects: - description: 'Comma-separated list of projects to test (e.g., "rocblas,hipblas" or "projects/rocblas"). Empty = run all tests.' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - -permissions: - contents: read - -jobs: - copy_prebuilt_stages: - name: Copy Prebuilt Stages - if: ${{ fromJSON(inputs.build_config).prebuilt_stages != '' && fromJSON(inputs.build_config).baseline_run_id != '' }} - runs-on: aws-linux-scale-rocm-prod-rc - permissions: - contents: read - id-token: write - container: - image: python:3.12-slim - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Install artifact manager dependencies - run: pip install boto3 - - - name: Configure AWS credentials for artifact uploads - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Copy prebuilt stage artifacts - env: - GITHUB_TOKEN: ${{ github.token }} - RELEASE_TYPE: ${{ inputs.release_type }} - run: | - python build_tools/artifact_manager.py copy \ - --source-run-id=${{ fromJSON(inputs.build_config).baseline_run_id }} \ - --source-repository=${{ fromJSON(inputs.build_config).baseline_repository }} \ - --stage="${{ fromJSON(inputs.build_config).prebuilt_stages }}" \ - --amdgpu-families="${{ fromJSON(inputs.build_config).dist_amdgpu_families }}" \ - --expand-family-to-targets - - build_multi_arch_stages: - name: Build Multi-Arch Stages - needs: copy_prebuilt_stages - if: ${{ !cancelled() && !failure() }} - uses: ./.github/workflows/multi_arch_build_portable_linux.yml - secrets: inherit - with: - matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }} - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }} - build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }} - build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }} - prebuilt_stages: ${{ fromJSON(inputs.build_config).prebuilt_stages }} - build_runs_on: ${{ fromJSON(inputs.build_config).build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - external_repo_config: ${{ inputs.external_repo_config }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - validate_artifact_structure: - needs: [build_multi_arch_stages] - name: Validate Artifact Structure - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/test_artifacts_structure.yml - with: - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - platform: linux - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - test_artifacts_per_family: - needs: [copy_prebuilt_stages, build_multi_arch_stages] - name: Test ${{ matrix.family_info.amdgpu_family }} - if: ${{ !failure() && !cancelled() }} - strategy: - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.build_config).per_family_info }} - uses: ./.github/workflows/test_artifacts.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - amdgpu_families: ${{ matrix.family_info.amdgpu_family }} - amdgpu_targets: ${{ matrix.family_info.amdgpu_targets }} - test_runs_on: ${{ matrix.family_info.test-runs-on }} - test_type: ${{ inputs.test_type }} - # Per-family test_labels override global test_labels when specified - # Convert array to JSON string for workflow input compatibility - test_labels: ${{ matrix.family_info.test_labels_for_family && toJSON(matrix.family_info.test_labels_for_family) || inputs.test_labels }} - sanity_check_only_for_family: ${{ matrix.family_info.sanity_check_only_for_family }} - changed_projects: ${{ inputs.changed_projects }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - - build_native_deb_packages: - needs: [build_multi_arch_stages] - name: Build DEB Packages - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - rocm_version: ${{ inputs.rocm_deb_package_version || inputs.rocm_package_version }} - native_package_type: deb - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_native_rpm_packages: - needs: [build_multi_arch_stages] - name: Build RPM Packages - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - rocm_version: ${{ inputs.rocm_rpm_package_version || inputs.rocm_package_version }} - native_package_type: rpm - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_native_packages_ubuntu2404: - needs: [build_native_deb_packages] - name: Test DEB Install - ubuntu2404 - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/test_native_linux_packages_install.yml - with: - package_install_url: ${{ needs.build_native_deb_packages.outputs.package_repository_url }} - release_type: ci - os_profile: ubuntu2404 - artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - test_type: sanity - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_native_packages_rhel8: - needs: [build_native_rpm_packages] - name: Test RPM Install - rhel8 - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/test_native_linux_packages_install.yml - with: - package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }} - release_type: ci - os_profile: rhel8 - artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - test_type: sanity - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_native_packages_rhel10: - needs: [build_native_rpm_packages] - name: Test RPM Install - rhel10 - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/test_native_linux_packages_install.yml - with: - package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }} - release_type: ci - os_profile: rhel10 - artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - test_type: sanity - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_native_packages_sles16: - needs: [build_native_rpm_packages] - name: Test RPM Install - sles16 - if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_native_linux == true }} - uses: ./.github/workflows/test_native_linux_packages_install.yml - with: - package_install_url: ${{ needs.build_native_rpm_packages.outputs.package_repository_url }} - release_type: ci - os_profile: sles16 - artifact_group: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - test_type: sanity - build_variant: ${{ fromJSON(inputs.build_config).build_variant_label }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_python_packages: - needs: [build_multi_arch_stages] - name: Build Python Packages - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/build_portable_linux_python_packages.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - linux_amdgpu_families: ${{ fromJSON(inputs.build_config).linux_amdgpu_families }} - windows_amdgpu_families: ${{ fromJSON(inputs.build_config).windows_amdgpu_families }} - multiarch_index: true - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_python_packages_per_family: - needs: [build_python_packages] - name: Test Python - if: ${{ !failure() && !cancelled() && toJSON(fromJSON(inputs.build_config).test_python_packages_matrix) != '[]' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.build_config).test_python_packages_matrix }} - uses: ./.github/workflows/test_rocm_wheels.yml - with: - amdgpu_family: ${{ matrix.amdgpu_family }} - test_runs_on: ${{ matrix.test_runs_on }} - # TODO: Simplify to just `needs.build_python_packages.outputs.package_find_links_url` - # once kpack split is always enabled. - package_find_links_url: >- - ${{ - needs.build_python_packages.outputs.kpack_split == 'true' - && needs.build_python_packages.outputs.package_find_links_url - || format('{0}/{1}/index.html', - needs.build_python_packages.outputs.package_find_links_url, - matrix.amdgpu_family) - }} - python_version: ${{ matrix.python_version }} - rocm_version: ${{ inputs.rocm_package_version }} - container_image_name: ${{ matrix.container_image_name }} - container_image_url: ${{ matrix.container_image_url }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - # Multi-arch PyTorch build: one fat wheel covering every target, then - # split into a host wheel + per-gfx device wheels via rocm-kpack. - # KPACK_SPLIT_ARTIFACTS is always ON in multi-arch CI (FLAGS.cmake - # default; build_configure.py's opt-out only fires in single-arch CI), - # so the split path is unconditional here. The split itself runs inline - # in multi_arch_build_portable_linux_pytorch_wheels_ci.yml based on - # inputs.kpack_split. - build_pytorch_wheel_fat: - needs: [build_python_packages] - name: Build PyTorch (fat + split) - if: >- - ${{ - !failure() && - !cancelled() && - fromJSON(inputs.build_config).build_pytorch == true && - toJSON(fromJSON(inputs.build_config).pytorch_build_matrix) != '[]' - }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.build_config).pytorch_build_matrix }} - uses: ./.github/workflows/multi_arch_build_portable_linux_pytorch_wheels_ci.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - python_version: ${{ matrix.python_version }} - pytorch_git_ref: ${{ matrix.pytorch_git_ref }} - # The reusable workflow expands this to gfx targets via - # cmake/therock_amdgpu_targets.cmake. - amdgpu_families: ${{ matrix.amdgpu_families }} - kpack_split: ${{ needs.build_python_packages.outputs.kpack_split }} - rocm_package_find_links_url: ${{ needs.build_python_packages.outputs.package_find_links_url }} - rocm_version: ${{ inputs.rocm_package_version }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_jax_wheels: - needs: [build_python_packages] - name: Build JAX Wheels - if: >- - ${{ - !failure() && - !cancelled() && - fromJSON(inputs.build_config).build_jax == true && - toJSON(fromJSON(inputs.build_config).jax_build_matrix) != '[]' - }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.build_config).jax_build_matrix }} - uses: ./.github/workflows/multi_arch_build_linux_jax_wheels_ci.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - python_version: ${{ matrix.python_version }} - jax_repository: ${{ matrix.jax_repository }} - gfx_arch: ${{ matrix.gfx_arch }} - jax_ref: ${{ matrix.jax_ref }} - rocm_version: ${{ inputs.rocm_package_version }} - rocm_package_find_links_url: ${{ needs.build_python_packages.outputs.package_find_links_url }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write diff --git a/.github/workflows/multi_arch_ci_windows.yml b/.github/workflows/multi_arch_ci_windows.yml deleted file mode 100644 index 3dc4b38555..0000000000 --- a/.github/workflows/multi_arch_ci_windows.yml +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Multi-Arch CI - Windows - -on: - workflow_call: - inputs: - build_config: - type: string - description: >- - JSON object with build configuration for this platform. Fields: - artifact_group, per_family_info, dist_amdgpu_families, - build_variant_label, build_variant_cmake_preset, - build_variant_suffix, build_pytorch, - build_native_linux, test_python_packages_matrix, - pytorch_build_matrix, - prebuilt_stages, baseline_run_id. - test_labels: - type: string - rocm_package_version: - type: string - test_type: - type: string - release_type: - description: 'Release type: "ci", "dev", "nightly", or "prerelease".' - type: string - default: ci - external_repo_config: - description: "JSON config for external repo checkout (repository, ref, checkout_path, source_package)" - type: string - default: "" - changed_projects: - description: 'Comma-separated list of projects to test (e.g., "rocblas,hipblas" or "projects/rocblas"). Empty = run all tests.' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - -permissions: - contents: read - -jobs: - copy_prebuilt_stages: - name: Copy Prebuilt Stages - if: ${{ fromJSON(inputs.build_config).prebuilt_stages != '' && fromJSON(inputs.build_config).baseline_run_id != '' }} - # TODO: Consider running on a Linux runner with --platform=windows to - # avoid Windows runner setup overhead (setup-python ~51s). - runs-on: ${{ fromJSON(inputs.build_config).build_runs_on || 'azure-windows-scale-rocm' }} - defaults: - run: - shell: bash - permissions: - contents: read - id-token: write - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install artifact manager dependencies - run: pip install boto3 - - - name: Configure AWS credentials for artifact uploads - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Copy prebuilt stage artifacts - env: - GITHUB_TOKEN: ${{ github.token }} - RELEASE_TYPE: ${{ inputs.release_type }} - run: | - python build_tools/artifact_manager.py copy \ - --source-run-id=${{ fromJSON(inputs.build_config).baseline_run_id }} \ - --source-repository=${{ fromJSON(inputs.build_config).baseline_repository }} \ - --stage="${{ fromJSON(inputs.build_config).prebuilt_stages }}" \ - --amdgpu-families="${{ fromJSON(inputs.build_config).dist_amdgpu_families }}" \ - --expand-family-to-targets - - build_multi_arch_stages: - name: Build Multi-Arch Stages - needs: copy_prebuilt_stages - if: ${{ !cancelled() && !failure() }} - uses: ./.github/workflows/multi_arch_build_windows.yml - secrets: inherit - with: - matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }} - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }} - build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }} - build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }} - prebuilt_stages: ${{ fromJSON(inputs.build_config).prebuilt_stages }} - build_runs_on: ${{ fromJSON(inputs.build_config).build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - external_repo_config: ${{ inputs.external_repo_config }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - validate_artifact_structure: - needs: [build_multi_arch_stages] - name: Validate Artifact Structure - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/test_artifacts_structure.yml - with: - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - platform: windows - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - test_artifacts_per_family: - needs: [copy_prebuilt_stages, build_multi_arch_stages] - name: Test ${{ matrix.family_info.amdgpu_family }} - if: ${{ !failure() && !cancelled() }} - strategy: - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.build_config).per_family_info }} - uses: ./.github/workflows/test_artifacts.yml - with: - # Use architecture-specific artifact group for fetching per-arch artifacts - artifact_group: ${{ matrix.family_info.amdgpu_family }} - amdgpu_families: ${{ matrix.family_info.amdgpu_family }} - amdgpu_targets: ${{ matrix.family_info.amdgpu_targets }} - test_runs_on: ${{ matrix.family_info.test-runs-on }} - test_type: ${{ inputs.test_type }} - test_labels: ${{ inputs.test_labels }} - sanity_check_only_for_family: ${{ matrix.family_info.sanity_check_only_for_family }} - changed_projects: ${{ inputs.changed_projects }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - build_python_packages: - needs: [build_multi_arch_stages] - name: Build Python Packages - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/build_windows_python_packages.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - linux_amdgpu_families: ${{ fromJSON(inputs.build_config).linux_amdgpu_families }} - windows_amdgpu_families: ${{ fromJSON(inputs.build_config).windows_amdgpu_families }} - multiarch_index: true - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - test_python_packages_per_family: - needs: [build_python_packages] - name: Test Python - if: ${{ !failure() && !cancelled() && toJSON(fromJSON(inputs.build_config).test_python_packages_matrix) != '[]' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.build_config).test_python_packages_matrix }} - uses: ./.github/workflows/test_rocm_wheels.yml - with: - amdgpu_family: ${{ matrix.amdgpu_family }} - test_runs_on: ${{ matrix.test_runs_on }} - # TODO: Simplify to just `needs.build_python_packages.outputs.package_find_links_url` - # once kpack split is always enabled. - package_find_links_url: >- - ${{ - needs.build_python_packages.outputs.kpack_split == 'true' - && needs.build_python_packages.outputs.package_find_links_url - || format('{0}/{1}/index.html', - needs.build_python_packages.outputs.package_find_links_url, - matrix.amdgpu_family) - }} - python_version: ${{ matrix.python_version }} - rocm_version: ${{ inputs.rocm_package_version }} - container_image_name: ${{ matrix.container_image_name }} - container_image_url: ${{ matrix.container_image_url }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - # Multi-arch PyTorch build: one fat wheel covering every target, then - # split into a host wheel + per-gfx device wheels via rocm-kpack. - # KPACK_SPLIT_ARTIFACTS is always ON in multi-arch CI (FLAGS.cmake default), - # so the split path is unconditional here. The split itself runs inline - # in multi_arch_build_windows_pytorch_wheels_ci.yml based on - # inputs.kpack_split. - build_pytorch_wheel_fat: - needs: [build_python_packages] - name: Build PyTorch (fat + split) - if: >- - ${{ - !failure() && - !cancelled() && - fromJSON(inputs.build_config).build_pytorch == true && - toJSON(fromJSON(inputs.build_config).pytorch_build_matrix) != '[]' - }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.build_config).pytorch_build_matrix }} - uses: ./.github/workflows/multi_arch_build_windows_pytorch_wheels_ci.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - python_version: ${{ matrix.python_version }} - pytorch_git_ref: ${{ matrix.pytorch_git_ref }} - # The reusable workflow expands this to gfx targets via - # cmake/therock_amdgpu_targets.cmake. - amdgpu_families: ${{ matrix.amdgpu_families }} - kpack_split: ${{ needs.build_python_packages.outputs.kpack_split }} - rocm_package_find_links_url: ${{ needs.build_python_packages.outputs.package_find_links_url }} - rocm_version: ${{ inputs.rocm_package_version }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write diff --git a/.github/workflows/multi_arch_release.yml b/.github/workflows/multi_arch_release.yml deleted file mode 100644 index e757cbfaf8..0000000000 --- a/.github/workflows/multi_arch_release.yml +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# This workflow is the entry point for multi-arch releases. -# -# For manual dev releases, trigger via workflow_dispatch on TheRock. -# For scheduled nightly releases, rockrel will call this via workflow_call -# (TODO: pass required inputs through to setup job explicitly as needed). -# -# If we want per-platform status badges, independent retryability, or -# isolation from cross-platform runner queue issues, we can switch to -# dispatching each platform job via workflow_dispatch. That will also -# let us have this pre-trigger job freeze commits/versions so re-runs use -# consistent values, see https://github.com/ROCm/TheRock/issues/1236. - -name: Multi-Arch Release - -on: - workflow_call: - inputs: - release_type: - description: 'Release type: "dev", "nightly", or "prerelease".' - type: string - required: true - prerelease_version: - description: Prerelease version number such as '2'. This gets appended to the computed version after 'rc', like '7.10.0rc2' - type: string - default: "" - # For workflow_call, default to empty and trust the calling workflow to be explicit or pass "all". - linux_amdgpu_families: - description: 'Comma-separated list of Linux GPU families. "all" = all, "none" or empty = skip.' - type: string - default: "" - windows_amdgpu_families: - description: 'Comma-separated list of Windows GPU families. "all" = all, "none" or empty = skip.' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly Linux PyTorch wheel builds complete. - type: boolean - default: false - build_pytorch: - description: "Build PyTorch wheels" - type: boolean - default: true - build_jax: - description: "Build JAX wheels" - type: boolean - default: true - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - workflow_dispatch: - inputs: - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - prerelease_version: - description: Prerelease version number such as '2'. This gets appended to the computed version after 'rc', like '7.10.0rc2' - type: string - default: "" - # For workflow_dispatch, default to a single GPU family to limit CI load for most workflow runs. - # Note: empty string will be converted to the default by github, so we need an explicit "none". - linux_amdgpu_families: - description: 'Comma-separated list of Linux GPU families. "all" = all, "none" = skip platform.' - type: string - default: "gfx94x" - windows_amdgpu_families: - description: 'Comma-separated list of Windows GPU families. "all" = all, "none" = skip platform.' - type: string - default: "gfx110x" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly Linux PyTorch wheel builds complete. - type: boolean - default: false - build_pytorch: - description: "Build PyTorch wheels" - type: boolean - default: false - build_jax: - description: "Build JAX wheels" - type: boolean - default: false - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - -permissions: - contents: read - -run-name: >- - Multi-Arch Release (${{ inputs.release_type }}) - | Linux: ${{ inputs.linux_amdgpu_families || 'none' }} - | Windows: ${{ inputs.windows_amdgpu_families || 'none' }} - -jobs: - setup: - uses: ./.github/workflows/setup_multi_arch.yml - with: - build_variant: "release" - release_type: ${{ inputs.release_type }} - prerelease_version: ${{ inputs.prerelease_version }} - build_pytorch: ${{ inputs.build_pytorch }} - build_jax: ${{ inputs.build_jax }} - python_version: ${{ inputs.python_version }} - linux_amdgpu_families: ${{ inputs.linux_amdgpu_families }} - windows_amdgpu_families: ${{ inputs.windows_amdgpu_families }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - linux_release: - name: Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: ${{ needs.setup.outputs.linux_build_config != '' }} - uses: ./.github/workflows/multi_arch_release_linux.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.linux_build_config }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - rocm_deb_package_version: ${{ needs.setup.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ needs.setup.outputs.ref }} - run_full_pytorch_tests: ${{ inputs.run_full_pytorch_tests }} - build_pytorch: ${{ inputs.build_pytorch }} - build_jax: ${{ inputs.build_jax }} - python_version: ${{ inputs.python_version }} - permissions: - contents: read - actions: write # Added permission to trigger workflows - id-token: write # Added permission for AWS S3 upload - - windows_release: - name: Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: ${{ needs.setup.outputs.windows_build_config != '' }} - uses: ./.github/workflows/multi_arch_release_windows.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.windows_build_config }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ needs.setup.outputs.ref }} - build_pytorch: ${{ inputs.build_pytorch }} - python_version: ${{ inputs.python_version }} - permissions: - contents: read - actions: write # Added permission to trigger workflows - id-token: write # Added permission for AWS S3 upload diff --git a/.github/workflows/multi_arch_release_asan.yml b/.github/workflows/multi_arch_release_asan.yml deleted file mode 100644 index a10e3a9186..0000000000 --- a/.github/workflows/multi_arch_release_asan.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# This workflow is the entry point for ASAN releases. -# -# Modeled on multi_arch_release.yml but uses the ASAN build variant. - -name: Multi-Arch Release ASAN - -on: - workflow_call: - inputs: - release_type: - description: 'Release type: "dev", "nightly", or "prerelease".' - type: string - required: true - prerelease_version: - description: Prerelease version number such as '2'. This gets appended to the computed version after 'rc', like '7.10.0rc2' - type: string - default: "" - # For workflow_call, default to empty and trust the calling workflow to be explicit or pass "all". - linux_amdgpu_families: - description: 'Comma-separated list of Linux GPU families. "all" = all, "none" or empty = skip.' - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly Linux PyTorch wheel builds complete. - type: boolean - default: false - build_pytorch: - description: "Build PyTorch wheels" - type: boolean - default: true - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - workflow_dispatch: - inputs: - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - # For workflow_dispatch, default to a single GPU family to limit CI load for most workflow runs. - # Note: empty string will be converted to the default by github, so we need an explicit "none". - linux_amdgpu_families: - description: 'Comma-separated list of Linux GPU families. "all" = all, "none" = skip platform.' - type: string - default: "gfx94x" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly Linux PyTorch wheel builds complete. - type: boolean - default: false - build_pytorch: - description: "Build PyTorch wheels" - type: boolean - default: false - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - -permissions: - contents: read - -run-name: >- - Multi-Arch Release ASAN (${{ inputs.release_type || 'dev' }}) - | Linux: ${{ inputs.linux_amdgpu_families || 'all' }} - -jobs: - setup: - uses: ./.github/workflows/setup_multi_arch.yml - with: - build_variant: "asan" - release_type: ${{ inputs.release_type || 'dev' }} - prerelease_version: ${{ inputs.prerelease_version }} - linux_amdgpu_families: ${{ inputs.linux_amdgpu_families || 'all' }} - windows_amdgpu_families: "none" - build_pytorch: false - build_jax: false - python_version: ${{ inputs.python_version }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - linux_release: - name: Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} - needs: setup - if: ${{ needs.setup.outputs.linux_build_config != '' }} - uses: ./.github/workflows/multi_arch_release_linux.yml - secrets: inherit - with: - build_config: ${{ needs.setup.outputs.linux_build_config }} - rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }} - rocm_deb_package_version: ${{ needs.setup.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: ${{ needs.setup.outputs.rocm_rpm_package_version }} - test_type: ${{ needs.setup.outputs.test_type }} - release_type: ${{ inputs.release_type || 'dev' }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ needs.setup.outputs.ref }} - run_full_pytorch_tests: ${{ inputs.run_full_pytorch_tests }} - build_pytorch: ${{ inputs.build_pytorch }} - build_jax: false - python_version: ${{ inputs.python_version }} - build_variant: "asan" - permissions: - contents: read - actions: write - id-token: write diff --git a/.github/workflows/multi_arch_release_linux.yml b/.github/workflows/multi_arch_release_linux.yml deleted file mode 100644 index 8045735567..0000000000 --- a/.github/workflows/multi_arch_release_linux.yml +++ /dev/null @@ -1,282 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Multi-Arch Release Linux - -on: - workflow_call: - inputs: - build_config: - type: string - description: JSON object with build configuration for this platform. - rocm_package_version: - type: string - rocm_deb_package_version: - type: string - description: ROCm DEB package version for native packages (e.g., 8.0.0). - rocm_rpm_package_version: - type: string - description: ROCm RPM package version for native packages (e.g., 8.0.0). - test_type: - type: string - release_type: - description: 'Release type: "dev", "nightly", or "prerelease".' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly PyTorch wheel builds complete. - type: boolean - default: false - build_pytorch: - description: "Build PyTorch wheels (defaults to true for workflow_call)" - type: boolean - default: true - build_jax: - description: "Build JAX wheels (defaults to true for workflow_call)" - type: boolean - default: true - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - build_variant: - description: "Build variant (release, asan). Affects publish paths." - type: string - default: "release" - -permissions: - contents: read - -jobs: - build_artifacts: - name: Build Artifacts - uses: ./.github/workflows/multi_arch_build_portable_linux.yml - secrets: inherit - with: - matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }} - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }} - build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }} - build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }} - build_runs_on: ${{ fromJSON(inputs.build_config).build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_tarballs: - needs: [build_artifacts] - name: Build Tarballs - uses: ./.github/workflows/multi_arch_build_tarballs.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - platform: linux - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_python_packages: - needs: [build_artifacts] - name: Build Python Packages - uses: ./.github/workflows/build_portable_linux_python_packages.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - linux_amdgpu_families: ${{ fromJSON(inputs.build_config).linux_amdgpu_families }} - windows_amdgpu_families: ${{ fromJSON(inputs.build_config).windows_amdgpu_families }} - multiarch_index: true - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_native_deb_packages: - needs: [build_artifacts] - name: Build DEB Packages - uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_run_id: ${{ github.run_id }} - rocm_version: ${{ inputs.rocm_deb_package_version }} - native_package_type: deb - release_type: ${{ inputs.release_type }} - build_variant: ${{ inputs.build_variant }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_native_rpm_packages: - needs: [build_artifacts] - name: Build RPM Packages - uses: ./.github/workflows/multi_arch_build_native_linux_packages.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_run_id: ${{ github.run_id }} - rocm_version: ${{ inputs.rocm_rpm_package_version }} - native_package_type: rpm - release_type: ${{ inputs.release_type }} - build_variant: ${{ inputs.build_variant }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - publish_to_release_buckets: - needs: [build_tarballs, build_python_packages, build_native_deb_packages, build_native_rpm_packages] - name: Publish to Release Buckets - runs-on: ubuntu-24.04 - permissions: - id-token: write - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install Python requirements - run: pip install -r requirements.txt - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Copy to release buckets - run: | - python build_tools/github_actions/publish_rocm_to_release_buckets.py \ - --run-id="${{ github.run_id }}" \ - --platform=linux \ - --release-type="${{ inputs.release_type }}" \ - --kpack-split="${{ needs.build_python_packages.outputs.kpack_split }}" \ - --build-variant="${{ inputs.build_variant }}" - - trigger_test_artifacts_per_family: - needs: [build_artifacts] - name: Trigger Test Artifacts (${{ matrix.family_info.amdgpu_family }}) - runs-on: ubuntu-24.04 - permissions: - actions: write - strategy: - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.build_config).per_family_info }} - steps: - - name: Trigger test_artifacts - if: ${{ matrix.family_info.test-runs-on != '' }} - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: test_artifacts.yml - inputs: | - { "artifact_group": "${{ matrix.family_info.amdgpu_family }}", - "artifact_run_id": "${{ github.run_id }}", - "amdgpu_families": "${{ matrix.family_info.amdgpu_family }}", - "amdgpu_targets": "${{ matrix.family_info.amdgpu_targets }}", - "test_runs_on": "${{ matrix.family_info.test-runs-on }}", - "test_type": "${{ inputs.test_type }}", - "release_type": "${{ inputs.release_type }}", - "repository": "${{ inputs.repository }}", - "ref": "${{ inputs.ref }}", - "build_variant": "${{ inputs.build_variant }}" - } - - # Install tests are dispatched asynchronously (sanity) so publish - # is not blocked on GPU runner queues. See https://github.com/ROCm/TheRock/issues/5212 - trigger_test_native_install: - needs: [build_native_deb_packages, build_native_rpm_packages] - name: Trigger Native Install Test - ${{ matrix.os_profile }} - runs-on: ubuntu-24.04 - permissions: - actions: write - strategy: - fail-fast: false - matrix: - os_profile: [ubuntu2404, rhel8, rhel10, sles16] - steps: - - name: Trigger test_native_linux_packages_install - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: test_native_linux_packages_install.yml - inputs: | - { "package_install_url": "${{ matrix.os_profile == 'ubuntu2404' && needs.build_native_deb_packages.outputs.package_repository_url || needs.build_native_rpm_packages.outputs.package_repository_url }}", - "release_type": "${{ inputs.release_type }}", - "os_profile": "${{ matrix.os_profile }}", - "artifact_group": "${{ fromJSON(inputs.build_config).dist_amdgpu_families }}", - "test_type": "${{ inputs.test_type }}", - "build_variant": "${{ inputs.build_variant }}", - "repository": "${{ inputs.repository }}", - "ref": "${{ inputs.ref }}" - } - - - # TODO: Consider switch pulling ROCm packages from the CDN instead of the artifact - # bucket if the job order is not altered. - trigger_release_pytorch_wheels: - needs: [build_artifacts, build_python_packages] - name: Trigger Release PyTorch Wheels - if: ${{ inputs.build_pytorch != false && fromJSON(inputs.build_config).build_pytorch == true }} - runs-on: ubuntu-24.04 - permissions: - actions: write - steps: - - name: Trigger PyTorch Build - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: multi_arch_release_linux_pytorch_wheels.yml - inputs: | - { "amdgpu_families": "${{ fromJSON(inputs.build_config).dist_amdgpu_families }}", - "rocm_version": "${{ inputs.rocm_package_version }}", - "rocm_package_find_links_url": "${{ needs.build_python_packages.outputs.package_find_links_url }}", - "release_type": "${{ inputs.release_type }}", - "run_full_pytorch_tests": "${{ inputs.run_full_pytorch_tests }}", - "ref": "${{ inputs.ref || '' }}", - "python_version": "${{ inputs.python_version }}" - } - - trigger_release_jax_wheels: - needs: [build_artifacts, build_python_packages] - name: Trigger Release JAX Wheels - if: ${{ inputs.build_jax != false && fromJSON(inputs.build_config).build_jax == true }} - runs-on: ubuntu-24.04 - permissions: - actions: write - steps: - - name: Trigger JAX Build - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: multi_arch_release_linux_jax_wheels.yml - inputs: | - { - "test_amdgpu_family": "${{ contains(fromJSON(inputs.build_config).dist_amdgpu_families, 'gfx94X-dcgpu') && 'gfx94X-dcgpu' || '' }}", - "release_type": "${{ inputs.release_type }}", - "rocm_version": "${{ inputs.rocm_package_version }}", - "rocm_package_find_links_url": "${{ needs.build_python_packages.outputs.package_find_links_url }}", - "repository": "${{ inputs.repository || github.repository }}", - "ref": "${{ inputs.ref || '' }}", - "python_version": "${{ inputs.python_version }}" - } diff --git a/.github/workflows/multi_arch_release_linux_jax_wheels.yml b/.github/workflows/multi_arch_release_linux_jax_wheels.yml deleted file mode 100644 index fee0141ed2..0000000000 --- a/.github/workflows/multi_arch_release_linux_jax_wheels.yml +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-arch release entry point for Linux JAX wheels. -# -# This workflow owns the release matrix and delegates each matrix cell to the -# reusable single-build workflow. - -name: Multi-Arch Release Linux JAX Wheels - -on: - workflow_call: - inputs: - test_amdgpu_family: - description: "GPU family to use for JAX tests; empty means skip tests." - type: string - default: "" - release_type: - description: 'Release type: "dev", "nightly", or "prerelease".' - type: string - required: true - rocm_version: - description: "ROCm package version to build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout." - type: string - default: "" - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - workflow_dispatch: - inputs: - test_amdgpu_family: - description: "Representative GPU family to use for JAX tests." - type: string - default: "gfx94X-dcgpu" - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - rocm_version: - description: "ROCm package version to build against." - type: string - required: true - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout." - type: string - default: "" - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - -run-name: Release Linux JAX Wheels (${{ inputs.release_type }}, ${{ inputs.rocm_version }}, Multiarch) - -permissions: - contents: read -jobs: - setup_matrix: - runs-on: ubuntu-24.04 - outputs: - jax_matrix: ${{ steps.matrix.outputs.jax_matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Generate JAX matrix - id: matrix - run: | - python ./build_tools/github_actions/configure_jax_release_matrix.py \ - --python-versions="${{ inputs.python_version }}" - - build_jax_wheels: - needs: setup_matrix - name: Build | py ${{ matrix.python_version }} | jax ${{ matrix.jax_ref }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.setup_matrix.outputs.jax_matrix) }} - - permissions: - id-token: write - contents: read - uses: ./.github/workflows/multi_arch_build_linux_jax_wheels.yml - with: - test_amdgpu_family: ${{ inputs.test_amdgpu_family }} - python_version: ${{ matrix.python_version }} - jax_ref: ${{ matrix.jax_ref }} - jax_repository: ${{ matrix.jax_repository }} - gfx_arch: ${{ matrix.gfx_arch }} - rocm_version: ${{ inputs.rocm_version }} - rocm_package_find_links_url: ${{ inputs.rocm_package_find_links_url }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} diff --git a/.github/workflows/multi_arch_release_linux_pytorch_wheels.yml b/.github/workflows/multi_arch_release_linux_pytorch_wheels.yml deleted file mode 100644 index 6e97258dd2..0000000000 --- a/.github/workflows/multi_arch_release_linux_pytorch_wheels.yml +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-arch release entry point for Linux PyTorch wheels. -# -# This workflow owns the release matrix and delegates each matrix cell to the -# reusable single-build workflow. For convergence with CI and testing plans see: -# * https://github.com/ROCm/TheRock/issues/3332 -# * https://github.com/ROCm/TheRock/issues/3291 - -name: Multi-Arch Release Linux PyTorch Wheels - -on: - workflow_call: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx94X-dcgpu;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; selects the S3 bucket' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type ('none', 'sccache', or 'ccache')" - type: string - default: "sccache" - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly wheel builds complete. - type: boolean - default: false - workflow_dispatch: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx94X-dcgpu;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; developer-triggered jobs should use "dev"' - type: choice - options: - - dev - default: dev - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - run_full_pytorch_tests: - description: Run full PyTorch tests after scheduled nightly wheel builds complete. - type: boolean - default: false - -run-name: Release Linux PyTorch Wheels (${{ inputs.release_type }}, ${{ inputs.rocm_version }}, ${{ inputs.amdgpu_families }}) - -permissions: - contents: read - -jobs: - setup_matrix: - runs-on: ubuntu-24.04 - outputs: - pytorch_matrix: ${{ steps.matrix.outputs.pytorch_matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Generate PyTorch matrix - id: matrix - run: | - python ./build_tools/github_actions/configure_pytorch_release_matrix.py \ - --python-versions="${{ inputs.python_version }}" \ - --platform=linux \ - --release-type="${{ inputs.release_type }}" \ - --amdgpu-families="${{ inputs.amdgpu_families }}" - - build_pytorch_wheels: - needs: setup_matrix - name: Build | py ${{ matrix.python_version }} | torch ${{ matrix.pytorch_git_ref }} - if: ${{ needs.setup_matrix.outputs.pytorch_matrix != '[]' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.setup_matrix.outputs.pytorch_matrix) }} - permissions: - id-token: write - contents: read - # Granted so the reusable build workflow's dispatch_pytorch_wheels_full_test - # job can trigger the full test workflow (a reusable workflow's job cannot - # request more permissions than its caller grants). - actions: write - uses: ./.github/workflows/multi_arch_build_portable_linux_pytorch_wheels.yml - with: - amdgpu_families: ${{ matrix.amdgpu_families }} - python_version: ${{ matrix.python_version }} - pytorch_git_ref: ${{ matrix.pytorch_git_ref }} - rocm_version: ${{ inputs.rocm_version }} - rocm_package_find_links_url: ${{ inputs.rocm_package_find_links_url }} - rocm_package_index_url: ${{ inputs.rocm_package_index_url }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - cache_type: ${{ inputs.cache_type }} - # The reusable build workflow dispatches the full PyTorch test suite for - # each successfully built (nightly, gfx94X-dcgpu, py3.12) ref using that - # build's own torch_version output (#5777). - run_full_pytorch_tests: ${{ inputs.run_full_pytorch_tests }} diff --git a/.github/workflows/multi_arch_release_windows.yml b/.github/workflows/multi_arch_release_windows.yml deleted file mode 100644 index 0f8b7081d2..0000000000 --- a/.github/workflows/multi_arch_release_windows.yml +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Multi-Arch Release Windows - -on: - workflow_call: - inputs: - build_config: - type: string - description: JSON object with build configuration for this platform. - rocm_package_version: - type: string - test_type: - type: string - release_type: - description: 'Release type: "dev", "nightly", or "prerelease".' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - build_pytorch: - description: "Build PyTorch wheels (defaults to true for workflow_call)" - type: boolean - default: true - python_version: - description: "Single Python version for wheel builds (empty for full matrix)" - type: string - default: "" - -permissions: - contents: read - -jobs: - build_artifacts: - name: Build Artifacts - uses: ./.github/workflows/multi_arch_build_windows.yml - secrets: inherit - with: - matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }} - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }} - build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }} - build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }} - build_runs_on: ${{ fromJSON(inputs.build_config).build_runs_on }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_tarballs: - needs: [build_artifacts] - name: Build Tarballs - uses: ./.github/workflows/multi_arch_build_tarballs.yml - with: - dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - platform: windows - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - build_python_packages: - needs: [build_artifacts] - name: Build Python Packages - uses: ./.github/workflows/build_windows_python_packages.yml - with: - artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }} - amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }} - linux_amdgpu_families: ${{ fromJSON(inputs.build_config).linux_amdgpu_families }} - windows_amdgpu_families: ${{ fromJSON(inputs.build_config).windows_amdgpu_families }} - multiarch_index: true - package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - permissions: - contents: read - id-token: write - - publish_to_release_buckets: - needs: [build_tarballs, build_python_packages] - name: Publish to Release Buckets - runs-on: ubuntu-24.04 - permissions: - id-token: write - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install Python requirements - run: pip install -r requirements.txt - - - name: Configure AWS Credentials - uses: ./.github/actions/configure_aws_artifacts_credentials - with: - release_type: ${{ inputs.release_type }} - - - name: Copy to release buckets - run: | - python build_tools/github_actions/publish_rocm_to_release_buckets.py \ - --run-id="${{ github.run_id }}" \ - --platform=windows \ - --release-type="${{ inputs.release_type }}" \ - --kpack-split="${{ needs.build_python_packages.outputs.kpack_split }}" - - trigger_test_artifacts_per_family: - needs: [build_artifacts] - name: Trigger Test Artifacts (${{ matrix.family_info.amdgpu_family }}) - if: ${{ !failure() && !cancelled() }} - runs-on: ubuntu-24.04 - permissions: - actions: write - strategy: - fail-fast: false - matrix: - family_info: ${{ fromJSON(inputs.build_config).per_family_info }} - steps: - - name: Trigger test_artifacts - if: ${{ matrix.family_info.test-runs-on != '' }} - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: test_artifacts.yml - inputs: | - { "artifact_group": "${{ matrix.family_info.amdgpu_family }}", - "artifact_run_id": "${{ github.run_id }}", - "amdgpu_families": "${{ matrix.family_info.amdgpu_family }}", - "amdgpu_targets": "${{ matrix.family_info.amdgpu_targets }}", - "test_runs_on": "${{ matrix.family_info.test-runs-on }}", - "test_type": "${{ inputs.test_type }}", - "release_type": "${{ inputs.release_type }}", - "repository": "${{ inputs.repository }}", - "ref": "${{ inputs.ref }}" - } - - # TODO: Consider switch pulling ROCm packages from the CDN instead of the artifact - # bucket if the job order is not altered. - trigger_release_pytorch_wheels: - needs: [build_artifacts, build_python_packages] - name: Trigger Release PyTorch Wheels - if: ${{ inputs.build_pytorch != false && fromJSON(inputs.build_config).build_pytorch == true }} - runs-on: ubuntu-24.04 - permissions: - actions: write - steps: - - name: Trigger PyTorch Build - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: multi_arch_release_windows_pytorch_wheels.yml - inputs: | - { "amdgpu_families": "${{ fromJSON(inputs.build_config).dist_amdgpu_families }}", - "rocm_version": "${{ inputs.rocm_package_version }}", - "rocm_package_find_links_url": "${{ needs.build_python_packages.outputs.package_find_links_url }}", - "release_type": "${{ inputs.release_type }}", - "ref": "${{ inputs.ref || '' }}", - "python_version": "${{ inputs.python_version }}" - } diff --git a/.github/workflows/multi_arch_release_windows_pytorch_wheels.yml b/.github/workflows/multi_arch_release_windows_pytorch_wheels.yml deleted file mode 100644 index 51a515b501..0000000000 --- a/.github/workflows/multi_arch_release_windows_pytorch_wheels.yml +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-arch release entry point for Windows PyTorch wheels. -# -# This workflow owns the release matrix and delegates each matrix cell to the -# reusable single-build workflow. For convergence with CI and testing plans see: -# * https://github.com/ROCm/TheRock/issues/3332 -# * https://github.com/ROCm/TheRock/issues/3291 - -name: Multi-Arch Release Windows PyTorch Wheels - -on: - workflow_call: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx110X-all;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; selects the S3 bucket' - type: string - required: true - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type ('none', 'sccache', or 'ccache')" - type: string - default: "sccache" - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - workflow_dispatch: - inputs: - amdgpu_families: - description: >- - Semicolon-separated list of AMD GPU families to build PyTorch for - (e.g. 'gfx110X-all;gfx120X-all'). Expanded to gfx targets at - build time via cmake/therock_amdgpu_targets.cmake. - type: string - required: true - rocm_version: - description: ROCm package version to install and build against (e.g. 7.10.0.dev0) - type: string - required: true - rocm_package_find_links_url: - description: URL for pip --find-links to install ROCm packages (flat page; use this OR rocm_package_index_url) - type: string - default: "" - rocm_package_index_url: - description: URL for pip --index-url to install ROCm packages (PEP 503 index, e.g. repo.amd.com; use this OR rocm_package_find_links_url) - type: string - default: "" - release_type: - description: 'Release type; developer-triggered jobs should use "dev"' - type: choice - options: - - dev - default: dev - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - cache_type: - description: "Compiler cache type" - type: choice - options: - - none - - sccache - - ccache - default: sccache - python_version: - description: "Single Python version to build (empty for full matrix)" - type: string - default: "" - -run-name: Release Windows PyTorch Wheels (${{ inputs.release_type }}, ${{ inputs.rocm_version }}, ${{ inputs.amdgpu_families }}) - -permissions: - contents: read - -jobs: - setup_matrix: - runs-on: ubuntu-24.04 - outputs: - pytorch_matrix: ${{ steps.matrix.outputs.pytorch_matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Generate PyTorch matrix - id: matrix - run: | - python ./build_tools/github_actions/configure_pytorch_release_matrix.py \ - --python-versions="${{ inputs.python_version }}" \ - --platform=windows \ - --release-type="${{ inputs.release_type }}" \ - --amdgpu-families="${{ inputs.amdgpu_families }}" - - build_pytorch_wheels: - needs: setup_matrix - name: Build | py ${{ matrix.python_version }} | torch ${{ matrix.pytorch_git_ref }} - if: ${{ needs.setup_matrix.outputs.pytorch_matrix != '[]' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.setup_matrix.outputs.pytorch_matrix) }} - permissions: - id-token: write - contents: read - uses: ./.github/workflows/multi_arch_build_windows_pytorch_wheels.yml - with: - amdgpu_families: ${{ matrix.amdgpu_families }} - python_version: ${{ matrix.python_version }} - pytorch_git_ref: ${{ matrix.pytorch_git_ref }} - rocm_version: ${{ inputs.rocm_version }} - rocm_package_find_links_url: ${{ inputs.rocm_package_find_links_url }} - rocm_package_index_url: ${{ inputs.rocm_package_index_url }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - cache_type: ${{ inputs.cache_type }} diff --git a/.github/workflows/notify_quartz.yml b/.github/workflows/notify_quartz.yml deleted file mode 100644 index ae73c605d1..0000000000 --- a/.github/workflows/notify_quartz.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Notify Quartz - -on: - workflow_call: - inputs: - run_phase: - description: >- - Phase of the workflow run. Must be exactly one of: - 'started' (run is in progress) or 'completed' (run has finished). - required: true - type: string - run_conclusion: - description: >- - Overall conclusion for the calling workflow - (success / failure / cancelled). Optional: when omitted and - run_phase=completed, notify_quartz.py derives it from the per-job - `result` values found in workflow_captured_outputs (i.e. the - toJSON(needs) blob below). Pass an explicit value here only when - you need to override the derived result (e.g. mark a run as - failure even though every dependency reported success). - required: false - type: string - default: "" - reporting_workflow: - description: >- - Filename of the workflow that is reporting (e.g. - 'multi_arch_build_portable_linux.yml'). Each caller passes its own - filename as a literal so nested reusable workflows self-identify - (they otherwise share the top-level caller's github.workflow context). - The receiver keys on this filename to classify the leaf/orchestrator. - required: false - type: string - default: "" - workflow_inputs: - description: "JSON string of workflow inputs captured at dispatch time via toJSON(inputs)" - required: true - type: string - workflow_captured_outputs: - description: >- - JSON object of caller-supplied job context. Canonical pattern is - toJSON(needs) from a `notify_completed` job whose `needs:` lists - every job whose result/outputs you want recorded; the receiver then - has per-job `result` + `outputs` without any plucking on the caller - side. NOT a dump of env / vars / secrets: anything passed here is - stored verbatim in Quartz, so the caller is responsible for not - including secrets. - required: false - type: string - default: "{}" - quartz_repo: - description: "Target Quartz repository (owner/repo)" - required: false - type: string - default: "ROCm/Quartz" - quartz_workflow_file: - description: "Filename of the Quartz ingest workflow to dispatch to" - required: false - type: string - default: "receive_therock_data.yml" - quartz_workflow_ref: - description: "Branch in Quartz repo where the ingest workflow lives" - required: false - type: string - default: "main" - -permissions: - contents: read - -jobs: - notify: - name: Quartz - ${{ inputs.run_phase }} ${{ inputs.reporting_workflow }} - runs-on: ubuntu-24.04 - timeout-minutes: 10 - continue-on-error: true - steps: - - name: Checkout TheRock - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/TheRock - - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.GH_APP_HAULY_ID }} - private-key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} - repositories: ${{ inputs.quartz_repo }} - - name: Notify Quartz - env: - QUARTZ_HAULY_TOKEN: ${{ steps.app-token.outputs.token }} - WORKFLOW_INPUTS: ${{ inputs.workflow_inputs }} - WORKFLOW_CAPTURED_OUTPUTS: ${{ inputs.workflow_captured_outputs }} - REPORTING_WORKFLOW: ${{ inputs.reporting_workflow }} - RUN_CONCLUSION: ${{ inputs.run_conclusion }} - RUN_PHASE: ${{ inputs.run_phase }} - QUARTZ_REPO: ${{ inputs.quartz_repo }} - QUARTZ_WORKFLOW_FILE: ${{ inputs.quartz_workflow_file }} - QUARTZ_WORKFLOW_REF: ${{ inputs.quartz_workflow_ref }} - run: python build_tools/github_actions/notify_quartz.py diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index bba1a39a31..0000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -permissions: - contents: read - -jobs: - pre-commit: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/pre_commit_security.yml b/.github/workflows/pre_commit_security.yml deleted file mode 100644 index 08aa3cb84d..0000000000 --- a/.github/workflows/pre_commit_security.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -# Pre-commit security gate: fast, PR-side security scanners that run on -# every pull request. The security counterpart to `pre-commit.yml`. -# Sibling scanners can be added as additional jobs in this file as they -# come online, keeping all PR-side security gates discoverable in one -# place. - -name: Pre commit security - -on: - pull_request: - -permissions: - contents: read - -jobs: - # Scan commits introduced by this PR (`scan_mode: changed`, the - # reusable workflow's default). CSV report is uploaded as a build - # artifact and printed to the job summary so reviewers can browse - # findings directly from the PR's Checks tab. - gitleaks: - uses: ./.github/workflows/gitleaks.yml - with: - report_formats: csv diff --git a/.github/workflows/publish_build_manylinux_x86_64.yml b/.github/workflows/publish_build_manylinux_x86_64.yml deleted file mode 100644 index defde83be3..0000000000 --- a/.github/workflows/publish_build_manylinux_x86_64.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish build_manylinux_x86_64 images -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'stage/docker/**' - paths: - - dockerfiles/build_manylinux_x86_64*.Dockerfile - - .github/workflows/publish_build_manylinux_x86_64.yml - -permissions: - contents: read - packages: write - -jobs: - publish_build_manylinux_x86_64: - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: build_manylinux_x86_64 - DOCKER_IMAGE_NAME: therock_build_manylinux_x86_64 diff --git a/.github/workflows/publish_dockerfile.yml b/.github/workflows/publish_dockerfile.yml deleted file mode 100644 index c5b22d53df..0000000000 --- a/.github/workflows/publish_dockerfile.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish TheRock Docker image -on: - workflow_call: - inputs: - DOCKER_FILE_NAME: - type: string - DOCKER_IMAGE_NAME: - type: string - -permissions: - contents: read - -jobs: - build-and-push-image: - runs-on: ubuntu-24.04 - env: - REGISTRY: ghcr.io - IMAGE_NAME: ROCm/${{ inputs.DOCKER_IMAGE_NAME }} - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Log in to the Container registry - uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Sanitization of tag names is done automatically by the metadata-action - - name: Determine Docker tag - id: tag - run: | - ref="${{ github.ref_name }}" - if [[ "$ref" == stage/docker/* ]]; then - suffix="${ref#stage/docker/}" - echo "TAG_SUFFIX=stage-${suffix}" >> "$GITHUB_OUTPUT" - elif [[ "$ref" == "main" ]]; then - echo "TAG_SUFFIX=latest" >> "$GITHUB_OUTPUT" - else - echo "TAG_SUFFIX=${ref}" >> "$GITHUB_OUTPUT" - fi - - # Adds extra tags to the image, with the default tags from https://github.com/docker/metadata-action#tags-input - # The custom tag is for the branches prefixed with `stage/docker/`. - # For the default branch (i.e., main), the default behaviour remains and is labelled `latest`. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=schedule - type=ref,event=branch,enable={{is_default_branch}} - type=ref,event=tag - type=ref,event=pr - type=raw,value=${{ steps.tag.outputs.TAG_SUFFIX }} - - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: dockerfiles/ - file: dockerfiles/${{ inputs.DOCKER_FILE_NAME }}.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/publish_no_rocm_image_ubi10.yml b/.github/workflows/publish_no_rocm_image_ubi10.yml deleted file mode 100644 index c44e81c5c9..0000000000 --- a/.github/workflows/publish_no_rocm_image_ubi10.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish no_rocm_image_ubi10 image -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'stage/docker/**' - paths: - - dockerfiles/no_rocm_image_ubi10.Dockerfile - - .github/workflows/publish_no_rocm_image_ubi10.yml - -permissions: - contents: read - packages: write - -jobs: - publish_no_rocm_image_ubi10: - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubi10 - DOCKER_IMAGE_NAME: no_rocm_image_ubi10 diff --git a/.github/workflows/publish_no_rocm_image_ubuntu24_04.yml b/.github/workflows/publish_no_rocm_image_ubuntu24_04.yml deleted file mode 100644 index eaecce9bdb..0000000000 --- a/.github/workflows/publish_no_rocm_image_ubuntu24_04.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish no_rocm_image_ubuntu24_04 images -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'stage/docker/**' - paths: - - dockerfiles/no_rocm_image_ubuntu24_04*.Dockerfile - - .github/workflows/publish_no_rocm_image_ubuntu24_04.yml - -permissions: - contents: read - packages: write - -jobs: - publish_no_rocm_image_ubuntu24_04: - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04 - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04 - # The image no_rocm_image_ubuntu24_04_rocgdb is an extension of - # no_rocm_image_ubuntu24_04, so make sure we rebuild the _rocgdb - # version whenever the base version changes. - publish_no_rocm_image_ubuntu24_04_rocgdb: - needs: [publish_no_rocm_image_ubuntu24_04] - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_rocgdb - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_rocgdb - # image used in ocltst execution - publish_no_rocm_image_ubuntu24_04_ocl_rt: - needs: [publish_no_rocm_image_ubuntu24_04] - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_ocl_rt - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_ocl_rt - # The image no_rocm_image_ubuntu24_04_media is an extension of - # no_rocm_image_ubuntu24_04, so make sure we rebuild the _media - # version whenever the base version changes. - publish_no_rocm_image_ubuntu24_04_media: - needs: [publish_no_rocm_image_ubuntu24_04] - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_media - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_media - # The image no_rocm_image_ubuntu24_04_openmpi is an extension of - # no_rocm_image_ubuntu24_04, so make sure we rebuild the _openmpi - # version whenever the base version changes. - publish_no_rocm_image_ubuntu24_04_openmpi: - needs: [publish_no_rocm_image_ubuntu24_04] - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_openmpi - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_openmpi diff --git a/.github/workflows/publish_no_rocm_image_ubuntu24_04_ocl_rt.yml b/.github/workflows/publish_no_rocm_image_ubuntu24_04_ocl_rt.yml deleted file mode 100644 index 2854bce3ef..0000000000 --- a/.github/workflows/publish_no_rocm_image_ubuntu24_04_ocl_rt.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish no_rocm_image_ubuntu24_04_ocl_rt images -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'stage/docker/**' - paths: - - dockerfiles/no_rocm_image_ubuntu24_04_ocl_rt.Dockerfile - - .github/workflows/publish_no_rocm_image_ubuntu24_04_ocl_rt.yml - -permissions: - contents: read - packages: write - -jobs: - publish_no_rocm_image_ubuntu24_04_ocl_rt: - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_ocl_rt - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_ocl_rt diff --git a/.github/workflows/publish_no_rocm_image_ubuntu24_04_rocgdb.yml b/.github/workflows/publish_no_rocm_image_ubuntu24_04_rocgdb.yml deleted file mode 100644 index fdbd332e4e..0000000000 --- a/.github/workflows/publish_no_rocm_image_ubuntu24_04_rocgdb.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Publish no_rocm_image_ubuntu24_04_rocgdb images -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'stage/docker/**' - paths: - - dockerfiles/no_rocm_image_ubuntu24_04_rocgdb.Dockerfile - - .github/workflows/publish_no_rocm_image_ubuntu24_04_rocgdb.yml - -permissions: - contents: read - packages: write - -jobs: - publish_no_rocm_image_ubuntu24_04_rocgdb: - uses: ./.github/workflows/publish_dockerfile.yml - with: - DOCKER_FILE_NAME: no_rocm_image_ubuntu24_04_rocgdb - DOCKER_IMAGE_NAME: no_rocm_image_ubuntu24_04_rocgdb diff --git a/.github/workflows/setup_multi_arch.yml b/.github/workflows/setup_multi_arch.yml deleted file mode 100644 index aed1bf9b09..0000000000 --- a/.github/workflows/setup_multi_arch.yml +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Setup (Multi-Arch) - -on: - workflow_call: - inputs: - build_variant: - type: string - default: "release" - release_type: - description: >- - Release type: "ci", "dev", "nightly", or "prerelease"; controls - artifact bucket selection, IAM role, and package versioning. - type: string - default: ci - prerelease_version: - description: Prerelease version number such as '2'. This gets appended to the computed version after 'rc', like '7.10.0rc2' - type: string - default: "" - linux_amdgpu_families: - description: 'Comma-separated Linux GPU families. "all" = all, "none" or empty = skip.' - type: string - default: "" - windows_amdgpu_families: - description: 'Comma-separated Windows GPU families. "all" = all, "none" or empty = skip.' - type: string - default: "" - linux_test_labels: - description: "Comma-separated list of test labels for Linux." - type: string - default: "" - windows_test_labels: - description: "Comma-separated list of test labels for Windows." - type: string - default: "" - prebuilt_stages: - description: "Comma-separated build stages to skip (artifacts copied from baseline_run_id instead)." - type: string - default: "" - baseline_run_id: - description: "Workflow run ID to copy prebuilt stage artifacts from." - type: string - default: "" - stage_reuse_mode: - description: >- - Automatic stage-reuse mode (separate from the manual prebuilt_stages - input): "dry-run" (default) only reports which unaffected stages could - be reused; "reuse-stage" actually reuses them. - type: string - default: "dry-run" - stage_reuse_max_age_hours: - description: >- - Automatic stage reuse: reject baseline runs older than this many hours. default: "72" - type: string - default: "72" - stage_reuse_commit_history: - description: >- - Automatic stage reuse: number of recent branch commits to fetch for ancestry. default: "50" - type: string - default: "50" - build_pytorch: - description: "Build PyTorch wheels." - type: boolean - default: true - build_jax: - description: "Build JAX wheels." - type: boolean - default: false - python_version: - description: "Single Python version for wheel builds; empty uses the default matrix." - type: string - default: "" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - external_repo: - description: "JSON object with 'repository' (e.g., 'ROCm/rocm-libraries') and 'ref' fields for external repo builds." - type: string - default: "" - outputs: - enable_build_jobs: - description: Whether to enable build jobs. - value: ${{ jobs.setup.outputs.enable_build_jobs }} - linux_build_config: - description: JSON object with Linux build configuration, or empty string if skipped. - value: ${{ jobs.setup.outputs.linux_build_config }} - windows_build_config: - description: JSON object with Windows build configuration, or empty string if skipped. - value: ${{ jobs.setup.outputs.windows_build_config }} - test_type: - description: "The test type to run (quick, standard, comprehensive, full)." - value: ${{ jobs.setup.outputs.test_type }} - linux_test_labels: - description: ROCm projects to run Linux tests on. Optional filter. - value: ${{ jobs.setup.outputs.linux_test_labels }} - windows_test_labels: - description: ROCm projects to run Windows tests on. Optional filter. - value: ${{ jobs.setup.outputs.windows_test_labels }} - rocm_package_version: - description: ROCm package version (primarily for Python packages). - value: ${{ jobs.setup.outputs.rocm_package_version }} - rocm_deb_package_version: - description: ROCm DEB package version for native packages. - value: ${{ jobs.setup.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: - description: ROCm RPM package version for native packages. - value: ${{ jobs.setup.outputs.rocm_rpm_package_version }} - ref: - description: >- - Resolved commit SHA from checkout. Use this instead of the input ref - to ensure all downstream jobs use the same commit, even if the branch - HEAD moves during the workflow run. - value: ${{ jobs.setup.outputs.ref }} - external_repo_config: - description: "JSON config for external repo checkout (generated from external_repo_name)" - value: ${{ jobs.setup.outputs.external_repo_config }} - -permissions: - contents: read - -jobs: - setup: - runs-on: ubuntu-24.04 - outputs: - enable_build_jobs: ${{ steps.configure.outputs.enable_build_jobs }} - linux_build_config: ${{ steps.configure.outputs.linux_build_config }} - windows_build_config: ${{ steps.configure.outputs.windows_build_config }} - test_type: ${{ steps.configure.outputs.test_type }} - linux_test_labels: ${{ steps.configure.outputs.linux_test_labels }} - windows_test_labels: ${{ steps.configure.outputs.windows_test_labels }} - rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} - rocm_deb_package_version: ${{ steps.rocm_package_version.outputs.rocm_deb_package_version }} - rocm_rpm_package_version: ${{ steps.rocm_package_version.outputs.rocm_rpm_package_version }} - ref: ${{ steps.checkout.outputs.commit }} - external_repo_config: ${{ steps.external_repo_config.outputs.config_json }} - steps: - - name: Checking out repository - id: checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - # We need the parent commit to do a diff - fetch-depth: 2 - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - # Note: the script reads PR labels, the base ref, etc. from the event - # payload (GITHUB_EVENT_PATH). That is the original event that started - # the workflow run, which may not directly map to the "inputs" on - # the "workflow_call" trigger for this file. - - name: Configuring CI options - id: configure - env: - BUILD_VARIANT: ${{ inputs.build_variant }} - RELEASE_TYPE: ${{ inputs.release_type }} - BUILD_PYTORCH: ${{ inputs.build_pytorch }} - BUILD_JAX: ${{ inputs.build_jax }} - PYTHON_VERSION: ${{ inputs.python_version }} - LINUX_AMDGPU_FAMILIES: ${{ inputs.linux_amdgpu_families }} - WINDOWS_AMDGPU_FAMILIES: ${{ inputs.windows_amdgpu_families }} - LINUX_TEST_LABELS: ${{ inputs.linux_test_labels }} - WINDOWS_TEST_LABELS: ${{ inputs.windows_test_labels }} - PREBUILT_STAGES: ${{ inputs.prebuilt_stages }} - BASELINE_RUN_ID: ${{ inputs.baseline_run_id }} - STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }} - STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }} - STAGE_REUSE_MAX_AGE_HOURS: ${{ inputs.stage_reuse_max_age_hours }} - STAGE_REUSE_COMMIT_HISTORY: ${{ inputs.stage_reuse_commit_history }} - # For cross-repo artifact reuse, use repository input (defaults to GITHUB_REPOSITORY). - # External repos set repository to "ROCm/TheRock" to reuse TheRock's prebuilt artifacts. - THEROCK_REPOSITORY: ${{ inputs.repository }} - # Skip path filtering for external repos (git sha won't exist in TheRock) - SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }} - CI_CONFIG_PATH: ci-config - # External repo family overrides allow callers to add/modify GPU family configs - # (e.g., adding test runners for architectures not enabled in TheRock's default config) - EXTERNAL_FAMILY_OVERRIDES: ${{ inputs.external_repo != '' && toJSON(fromJSON(inputs.external_repo).family_overrides) || '' }} - run: python ./build_tools/github_actions/configure_multi_arch_ci.py - - # Note: Other scripts treat "dev releases" and "CI builds" differently. - # Here we set to a "dev" package version even for CI builds: - # - # release_type | package version | version example - # ------------ | --------------- | ------------------ - # ci | dev | 7.10.0.dev0+f689a8 - # dev | dev | 7.10.0.dev0+f689a8 - # nightly | nightly | 7.10.0a20251021 - # prerelease | prerelease | 7.10.0rc2 - # - # Other scripts upload "ci" and "dev release" packages to different - # buckets, use different ccache namespaces, etc. - - name: Compute package version - id: rocm_package_version - run: | - python ./build_tools/compute_rocm_package_version.py \ - --release-type=${{ inputs.release_type }} \ - --prerelease-version=${{ inputs.prerelease_version }} \ - --override-git-sha=${{ steps.checkout.outputs.commit }} - - - name: Detect external repo config - id: external_repo_config - if: ${{ inputs.external_repo != '' }} - run: | - python ./build_tools/github_actions/detect_external_repo_config.py \ - --external-repo-json='${{ inputs.external_repo }}' diff --git a/.github/workflows/test_artifacts.yml b/.github/workflows/test_artifacts.yml deleted file mode 100644 index f38035f617..0000000000 --- a/.github/workflows/test_artifacts.yml +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test Artifacts - -on: - workflow_dispatch: - inputs: - artifact_group: - type: string - artifact_run_id: - type: string - default: "" - amdgpu_families: - type: string - amdgpu_targets: - type: string - default: "" - test_runs_on: - type: string - benchmark_runs_on: - type: string - default: "" - sanity_check_only_for_family: - type: boolean - default: false - test_type: - type: string - test_labels: - type: string - run_extended_tests: - type: string - default: 'false' - windows_hip_rocr_tests: - type: string - default: 'false' - changed_projects: - description: 'Comma-separated list of projects to test (e.g., "rocblas,hipblas" or "projects/rocblas"). Empty = run all tests.' - type: string - default: "" - release_type: - description: 'Release type: "ci" or "dev".' - type: choice - options: - - ci - - dev - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - build_variant: - description: "Build variant (e.g., release, asan)." - type: choice - options: - - release - - asan - default: release - workflow_call: - inputs: - artifact_group: - type: string - artifact_run_id: - type: string - default: "" - amdgpu_families: - type: string - amdgpu_targets: - type: string - default: "" - test_runs_on: - type: string - benchmark_runs_on: - type: string - default: "" - sanity_check_only_for_family: - type: boolean - default: false - test_type: - type: string - test_labels: - type: string - run_extended_tests: - type: string - default: 'false' - windows_hip_rocr_tests: - type: string - default: 'false' - changed_projects: - description: 'Comma-separated list of projects to test (e.g., "rocblas,hipblas" or "projects/rocblas"). Empty = run all tests.' - type: string - default: "" - release_type: - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - build_variant: - description: "Build variant (e.g., release, asan)." - type: string - default: "release" - push: - branches: - - ADHOCBUILD - -permissions: - contents: read - -run-name: Test Artifacts (${{ inputs.amdgpu_families }}, ${{ inputs.release_type }}, ${{ inputs.test_type }}, ${{ inputs.build_variant }}, ${{ inputs.test_runs_on }}) - -jobs: - configure_test_matrix: - name: "Configure test matrix (${{ inputs.amdgpu_families }})" - # if there is a test machine available - if: ${{ inputs.test_runs_on != '' }} - runs-on: ubuntu-24.04 - outputs: - sanity_component: ${{ steps.configure.outputs.sanity_component }} - components: ${{ steps.configure.outputs.components }} - platform: ${{ steps.configure.outputs.platform }} - shard_arr: ${{ steps.configure.outputs.shard_arr }} - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: "Determine test dependencies" - id: test_deps - shell: bash - env: - CHANGED_PROJECTS: ${{ inputs.changed_projects }} - run: | - python ./test_tools/determine_rocm_test_dependencies.py \ - --changed-projects "$CHANGED_PROJECTS" \ - --gha-output - - - name: "Configuring CI options" - id: configure - env: - ARTIFACT_GROUP: ${{ inputs.artifact_group }} - AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} - TEST_TYPE: ${{ inputs.test_type }} - TEST_LABELS: ${{ inputs.test_labels }} - RUN_EXTENDED_TESTS: ${{ inputs.run_extended_tests }} - WINDOWS_HIP_ROCR_TESTS: ${{ inputs.windows_hip_rocr_tests }} - PROJECTS_TO_TEST: ${{ steps.test_deps.outputs.projects_to_test }} - CI_CONFIG_PATH: ci-config - BUILD_VARIANT: ${{ inputs.build_variant }} - run: | - python ./build_tools/github_actions/fetch_test_configurations.py \ - --platform=${{ contains(inputs.test_runs_on, 'windows') && 'windows' || 'linux' }} - - test_sanity_check: - name: 'Test Sanity Check' - needs: configure_test_matrix - uses: './.github/workflows/test_component.yml' - with: - artifact_run_id: ${{ inputs.artifact_run_id }} - artifact_group: ${{ inputs.artifact_group }} - amdgpu_families: ${{ inputs.amdgpu_families }} - amdgpu_targets: ${{ inputs.amdgpu_targets }} - # For workflow_dispatch, honor the explicit test_runs_on input if provided. - # Otherwise, use the runner selected by fetch_test_configurations.py. - test_runs_on: ${{ (github.event_name == 'workflow_dispatch' && inputs.test_runs_on) || fromJSON(needs.configure_test_matrix.outputs.sanity_component).test_runner }} - platform: ${{ needs.configure_test_matrix.outputs.platform }} - component: ${{ needs.configure_test_matrix.outputs.sanity_component }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - build_variant: ${{ inputs.build_variant }} - - test_components: - name: 'Test ${{ matrix.components.job_name }}' - needs: [test_sanity_check, configure_test_matrix] - # Run when: - # - sanity check passed - # - there are components to test - # - sanity_check_only_for_family is not set - if: >- - ${{ - !failure() && - !cancelled() && - needs.configure_test_matrix.outputs.components != '' && - needs.configure_test_matrix.outputs.components != '[]' && - !inputs.sanity_check_only_for_family - }} - strategy: - fail-fast: false - matrix: - components: ${{ fromJSON(needs.configure_test_matrix.outputs.components || '[]') }} - uses: './.github/workflows/test_component.yml' - secrets: inherit - with: - artifact_run_id: ${{ inputs.artifact_run_id }} - artifact_group: ${{ inputs.artifact_group }} - amdgpu_families: ${{ inputs.amdgpu_families }} - amdgpu_targets: ${{ inputs.amdgpu_targets }} - # Route to the appropriate runner: - # 0. For workflow_dispatch, honor the explicit test_runs_on input if provided - # 1. Benchmark components use the dedicated benchmark runner - # 2. Multi-GPU components use the multi-GPU runner - # 3. CPU-only components use standard GitHub runners (azure-linux-scale-rocm) - # 4. Per-component runner (set by fetch_test_configurations.py) - test_runs_on: >- - ${{ - (github.event_name == 'workflow_dispatch' && - inputs.test_runs_on != '' && - inputs.test_runs_on) || - (matrix.components.is_benchmark == true && - inputs.benchmark_runs_on != '' && - inputs.benchmark_runs_on) || - (matrix.components.multi_gpu_runner != '' && - matrix.components.multi_gpu_runner) || - (matrix.components.linux_cpu_runner == true && - 'azure-linux-scale-rocm') || - matrix.components.test_runner - }} - platform: ${{ needs.configure_test_matrix.outputs.platform }} - component: ${{ toJSON(matrix.components) }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - build_variant: ${{ inputs.build_variant }} diff --git a/.github/workflows/test_artifacts_structure.yml b/.github/workflows/test_artifacts_structure.yml deleted file mode 100644 index bbc3b6b76e..0000000000 --- a/.github/workflows/test_artifacts_structure.yml +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test Artifact Structure - -on: - workflow_dispatch: - inputs: - artifact_run_id: - type: string - description: "Run ID to fetch artifacts from (defaults to current run)" - default: "" - artifact_github_repo: - description: "GitHub repository for artifact_run_id" - type: string - default: ROCm/TheRock - amdgpu_families: - description: "amdgpu_families as a semicolon-separated list (e.g. 'gfx94X-dcgpu;gfx120X-all')" - type: string - default: "" - platform: - type: choice - description: "Platform to fetch artifacts for" - options: - - linux - - windows - default: "linux" - release_type: - description: 'Release type: "ci" or "dev"' - type: choice - options: - - ci - - dev - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - workflow_call: - inputs: - artifact_run_id: - type: string - default: "" - artifact_github_repo: - type: string - default: "" - amdgpu_families: - type: string - default: "" - platform: - type: string - default: "linux" - release_type: - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - -permissions: - contents: read - -jobs: - test_artifact_structure: - name: "Validate artifact structure" - runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod-rc' || 'ubuntu-24.04' }} - container: - image: python:3.12-slim - env: - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - PLATFORM: ${{ inputs.platform }} - THEROCK_ARTIFACTS_DIR: ${{ github.workspace }}/.download_cache - RELEASE_TYPE: ${{ inputs.release_type }} - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Install test requirements - run: | - pip install -r requirements-test.txt - - - name: Fetch artifact archives (no extract) - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python ./build_tools/artifact_manager.py fetch \ - --run-id="${{ env.ARTIFACT_RUN_ID }}" \ - --run-github-repo="${{ inputs.artifact_github_repo }}" \ - --stage=all \ - --amdgpu-families="${{ inputs.amdgpu_families }}" \ - --platform="${{ env.PLATFORM }}" \ - --output-dir="${GITHUB_WORKSPACE}" \ - --no-extract - # NOTE: artifact_manager.py appends /.download_cache to --output-dir - # in --no-extract mode, so pass workspace root here so that - # artifacts land in THEROCK_ARTIFACTS_DIR ($workspace/.download_cache). - - - name: Validate artifact structure - run: | - python -m pytest tests/test_artifact_structure.py \ - -v --log-cli-level=info --timeout=300 diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml deleted file mode 100644 index 84113f4a69..0000000000 --- a/.github/workflows/test_component.yml +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test component - -on: - workflow_call: - inputs: - artifact_run_id: - type: string - default: "" - artifact_group: - type: string - amdgpu_families: - type: string - amdgpu_targets: - type: string - default: "" - test_runs_on: - type: string - platform: - type: string - component: - type: string - release_type: - type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - build_variant: - description: "Build variant (e.g., release, asan). Used to skip tests incompatible with certain builds." - type: string - default: "release" - default_container_image: - type: string - default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" - - -permissions: - contents: read - -jobs: - test_component: - name: >- - Test ${{ fromJSON(inputs.component).job_name }} - (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }}) - (${{ inputs.amdgpu_families }}) - ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }} - runs-on: ${{ inputs.test_runs_on }} - continue-on-error: ${{ fromJSON(inputs.component).expect_failure == true }} - timeout-minutes: 210 - container: - # If the component has specified an alternate image, honor that option. - # Otherwise use the default image. - image: >- - ${{ inputs.platform == 'linux' && - (fromJSON(inputs.component).container_image || inputs.default_container_image) - || null - }} - # Container options are configured in build_tools/github_actions/fetch_test_configurations.py - # and include base options (ipc, user, ulimit, seccomp), GPU options (devices, groups), - # and any job-specific options - options: ${{ fromJSON(inputs.component).container_options }} - strategy: - fail-fast: false - matrix: - # The shard array is based on "total_shards" from "fetch_test_configurations.py" - # The test executable will shard based on the array. (ex: [1, 2, 3, 4] = four test shards) - shard: ${{ fromJSON(inputs.component).shard_arr }} - defaults: - run: - shell: bash - env: - VENV_DIR: ${{ github.workspace }}/.venv - ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - OUTPUT_ARTIFACTS_DIR: "./build" - THEROCK_BIN_DIR: "./build/bin" - AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} - AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} - ARTIFACT_GROUP: ${{ inputs.artifact_group }} - BUILD_VARIANT: ${{ inputs.build_variant }} - # Benchmark results database API endpoints for performance metrics collection - # NOTE: These secrets are only required for benchmark results submission in nightly CI runs. - # For PR/push workflows, secret retrieval will fail gracefully and benchmarks will skip API submission. - BENCHMARK_DB_URL: ${{ secrets.BENCHMARK_DB_URL }} - BENCHMARK_DB_FALLBACK_URL: ${{ secrets.BENCHMARK_DB_FALLBACK_URL }} - steps: - - name: "Fetch 'build_tools' from repository" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - sparse-checkout: build_tools - path: "prejob" - - - name: Pre-job cleanup processes on Windows - if: ${{ runner.os == 'Windows' }} - timeout-minutes: 5 - shell: powershell - run: . '${{ github.workspace }}\prejob\build_tools\github_actions\cleanup_processes.ps1' - - - name: Pre-job cleanup GPU processes on Linux - if: ${{ runner.os == 'Linux' }} - timeout-minutes: 5 - run: bash ./prejob/build_tools/github_actions/cleanup_processes.sh - - - name: Checking repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Run setup test environment workflow - timeout-minutes: 15 - env: - RETRY_THIS_STEP: "true" - STEP_TIMEOUT_MINUTES: "15" # should match timeout-minutes specified above at this step - uses: './.github/actions/setup_test_environment' - with: - ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }} - # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved - ARTIFACT_GROUP: ${{ inputs.amdgpu_families }} - AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} - OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }} - VENV_DIR: ${{ env.VENV_DIR }} - FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} - RELEASE_TYPE: ${{ inputs.release_type }} - - # safe.directory must be set before Runner Health Status - - name: Adjust git config - run: | - git config --global --add safe.directory $PWD - git config fetch.parallel 10 - - - name: Runner health status - run: | - python ./build_tools/health_status.py - - - name: Driver / GPU sanity check - timeout-minutes: 3 - run: | - python ./build_tools/print_driver_gpu_info.py - - - name: Setup Additional Requirements - if: ${{ fromJSON(inputs.component).additional_requirements_files != '' }} - run: | - python ./build_tools/install_additional_requirements.py \ - --requirements-files ${{ join(fromJSON(inputs.component).additional_requirements_files, ',') }} - - # TODO(#3755): Clean this up after final sanitizer environment variable flags are determined - - name: Enable sanitizer-specific test flags - if: ${{ contains(inputs.artifact_group, 'asan') }} - run: | - echo "ASAN_OPTIONS=detect_odr_violation=0:quarantine_size_mb=600" >> $GITHUB_ENV - echo "HSA_XNACK=1" >> $GITHUB_ENV - echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV - - # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription. - # Must be done at runtime since container env vars aren't in GitHub's env context. - - name: Set thread limits - run: | - if [[ -n "${KUBE_CPU_REQUEST:-}" ]]; then - CPU_COUNT="${KUBE_CPU_REQUEST%.*}" - echo "OPENBLAS_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV - echo "OMP_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV - echo "Thread limits set: OPENBLAS_NUM_THREADS=${CPU_COUNT}, OMP_NUM_THREADS=${CPU_COUNT} (from KUBE_CPU_REQUEST=${KUBE_CPU_REQUEST})" - else - echo "KUBE_CPU_REQUEST not set, thread limits not configured" - fi - - - name: Test - id: test - timeout-minutes: ${{ fromJSON(inputs.component).timeout_minutes }} - env: - SHARD_INDEX: ${{ matrix.shard }} - TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }} - TEST_TYPE: ${{ fromJSON(inputs.component).test_type }} - TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }} - ROCM_KPACK_DEBUG: "1" - # Windows hip-tests (PAL=1, ROCR=0) set this via matrix; other components - # leave it empty. #3587 - # An empty GPU_ENABLE_PAL is misread as ROCr on Windows (atoi("")==0), - # whose Windows memory path OOMs some tests (e.g. rocsparse bsr*0 graph_test). - # Default unset components to the platform backend, mirroring the runtime fix - # in ROCm/rocm-systems#8050 (Windows -> PAL, Linux -> ROCr). - GPU_ENABLE_PAL: ${{ fromJSON(inputs.component).gpu_enable_pal || (inputs.platform == 'windows' && '1' || '0') }} - RETRY_THIS_STEP: "true" - RETRY_COUNT: "1" - RETRY_DELAY: "7" - STEP_TIMEOUT_MINUTES: ${{ fromJSON(inputs.component).timeout_minutes }} - STEP_NAME: >- - Test ${{ fromJSON(inputs.component).job_name }} - (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }}) - (${{ inputs.amdgpu_families }}) - ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }} - run: | - ${{ fromJSON(inputs.component).test_script }} - - - name: Print test reproduction command - if: ${{ failure() && steps.test.outcome == 'failure' }} - run: | - python ./build_tools/github_actions/reproduce_test_failure.py \ - --run-id "${{ env.ARTIFACT_RUN_ID }}" \ - --repository "${{ github.repository }}" \ - --amdgpu-family "${{ inputs.amdgpu_families }}" \ - --amdgpu-targets "${{ inputs.amdgpu_targets }}" \ - --test-script "${{ fromJSON(inputs.component).test_script }}" \ - --shard-index "${{ matrix.shard }}" \ - --total-shards "${{ fromJSON(inputs.component).total_shards }}" \ - --test-type "${{ fromJSON(inputs.component).test_type }}" \ - --print-cmd \ - ${{ fromJSON(inputs.component).fetch_artifact_args && format('--fetch-artifact-args="{0}"', fromJSON(inputs.component).fetch_artifact_args) || '' }} - - - name: Post-job cleanup GPU processes on Linux - if: ${{ always() && runner.os == 'Linux' }} - timeout-minutes: 5 - run: bash ./build_tools/github_actions/cleanup_processes.sh - - - name: Post-job cleanup processes on Windows - if: ${{ always() && runner.os == 'Windows' }} - timeout-minutes: 5 - shell: powershell - run: . '${{ github.workspace }}\build_tools\github_actions\cleanup_processes.ps1' diff --git a/.github/workflows/test_jax_dockerfile.yml b/.github/workflows/test_jax_dockerfile.yml deleted file mode 100644 index 1ff7b40898..0000000000 --- a/.github/workflows/test_jax_dockerfile.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test JAX Wheels - -on: - workflow_dispatch: - inputs: - test_runs_on: - required: true - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - image_name: - required: true - description: JAX docker image to run tests with - type: string - jax_version: - description: Version of JAX to install - required: false - type: string - jax_plugin_branch: - required: true - description: JAX plugin branch to checkout - type: string - default: "rocm-jaxlib-v0.11.0" - - workflow_call: - inputs: - test_runs_on: - required: true - type: string - image_name: - required: true - description: JAX docker image to run tests with - type: string - jax_version: - description: Version of JAX to install instead of the one on the docker image - required: false - type: string - jax_plugin_branch: - description: JAX plugin branch to checkout to use for test scripts - type: string - default: "rocm-jaxlib-v0.11.0" - -permissions: - contents: read - -jobs: - test_wheels: - name: Test - runs-on: ${{ inputs.test_runs_on }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repo: rocm/rocm-jax - # TODO: Add steps for creating the JAX docker image with an install of TheRock and then running JAX tests on the container diff --git a/.github/workflows/test_linux_jax_wheels.yml b/.github/workflows/test_linux_jax_wheels.yml deleted file mode 100644 index 74900237a3..0000000000 --- a/.github/workflows/test_linux_jax_wheels.yml +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test Linux JAX Wheels - -on: - workflow_call: - inputs: - amdgpu_family: - required: true - type: string - release_type: - required: true - type: string - package_index_url: - description: Base CloudFront URL for the Python package index - required: true - type: string - rocm_version: - description: ROCm version (optional, informational) - required: false - type: string - tar_url: - description: URL to TheRock tarball to configure ROCm - required: true - type: string - python_version: - description: Python version(s) to test (e.g., "3.12") - required: true - type: string - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`." - type: string - jax_ref: - description: rocm-jax repository ref/branch to check out - required: false - type: string - default: "rocm-jaxlib-v0.11.0" - jax_version: - description: "Base JAX version (e.g. 0.8.2) for installing jax from PyPI. Extracted from built wheels by write_jax_versions.py." - required: false - type: string - jaxlib_version: - description: "jaxlib wheel version (e.g. 0.9.0+rocm7 or 0.8.2+rocm7.12.0). Extracted from built wheels by write_jax_versions.py." - required: false - type: string - jax_plugin_version: - description: "jax_rocm7_plugin wheel version. Extracted from built wheels by write_jax_versions.py." - required: false - type: string - jax_pjrt_version: - description: "jax_rocm7_pjrt wheel version. Extracted from built wheels by write_jax_versions.py." - required: false - type: string - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." - type: string - test_runs_on: - required: true - type: string - - workflow_dispatch: - inputs: - amdgpu_family: - type: choice - options: - - gfx101X-dgpu - - gfx103X-all - - gfx110X-all - - gfx1150 - - gfx1151 - - gfx1152 - - gfx1153 - - gfx120X-all - - gfx900 - - gfx90c - - gfx906 - - gfx908 - - gfx90a - - gfx94X-dcgpu - - gfx950-dcgpu - - gfx125X-dcgpu - default: gfx94X-dcgpu - release_type: - description: 'Release type; developer-triggered jobs should use "dev"' - type: choice - options: - - dev - default: dev - package_index_url: - description: Base CloudFront URL for the Python package index - required: true - type: string - default: https://rocm.nightlies.amd.com/v2-staging/ - rocm_version: - description: ROCm version - required: false - type: string - tar_url: - description: URL to TheRock tarball to configure ROCm - required: true - type: string - python_version: - description: Python version(s) to test (e.g., "3.12") - required: true - type: string - default: "3.12" - jax_ref: - description: rocm-jax repository ref/branch to check out - required: false - type: string - default: "rocm-jaxlib-v0.11.0" - jax_version: - description: "Base JAX version (e.g. 0.8.2). Leave empty to auto-detect from rocm-jax requirements." - required: false - type: string - jaxlib_version: - description: "jaxlib wheel version (e.g. 0.9.0+rocm7 or 0.8.2+rocm7.12.0). Leave empty to auto-compute from rocm_version." - required: false - type: string - jax_plugin_version: - description: "jax_rocm7_plugin wheel version. Leave empty to use jaxlib_version." - required: false - type: string - jax_pjrt_version: - description: "jax_rocm7_pjrt wheel version. Leave empty to use jaxlib_version." - required: false - type: string - test_runs_on: - description: Runner label to use. The selected runner should have a GPU supported by amdgpu_family - required: true - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." - type: string - -permissions: - contents: read - packages: read - -jobs: - test_jax_wheels: - name: Test JAX Wheels | ${{ inputs.amdgpu_family }} - runs-on: ${{ inputs.test_runs_on }} - container: - image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26 - # DNS workaround: These --dns entries are added to resolve CloudFront/runner DNS issues - # They should NOT be required for testing JAX and can be removed once DNS issues are resolved - # TODO: Remove these lines when issue is fixed - https://github.com/ROCm/TheRock/issues/3014 - options: >- - --device /dev/kfd - --device /dev/dri - --group-add render - --group-add video - --user root - --env-file /etc/podinfo/gha-gpu-isolation-settings - --dns 8.8.8.8 - --dns 8.8.4.4 - defaults: - run: - shell: bash - env: - VIRTUAL_ENV: ${{ github.workspace }}/.venv - AMDGPU_FAMILY: ${{ inputs.amdgpu_family }} - THEROCK_TAR_URL: ${{ inputs.tar_url }} - PYTHON_VERSION: ${{ inputs.python_version }} - WHEEL_INDEX_URL: ${{ inputs.package_index_url }}/${{ inputs.amdgpu_family }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Checkout rocm-jax (plugin + build scripts) - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: rocm-jax - repository: ROCm/rocm-jax - ref: ${{ inputs.jax_ref }} - - - name: Checkout JAX extended tests repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: jax - repository: ROCm/jax - ref: ${{ inputs.jax_ref }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - check-latest: true - - - name: System deps, venv configure - run: | - python3 -m venv "${VIRTUAL_ENV}" - echo "PATH=${VIRTUAL_ENV}/bin:${PATH}" >> "$GITHUB_ENV" - python3 build_tools/setup_venv.py "${VIRTUAL_ENV}" --activate-in-future-github-actions-steps - - - name: Install base JAX test requirements - run: | - # This script sets up the venv and activates it across steps; keep it consistent - pip install -r external-builds/jax/requirements-jax.txt - - - name: Configure ROCm from TheRock tarball - env: - ROCM_VERSION: ${{ inputs.rocm_version }} - AMDGPU_FAMILY: ${{ inputs.amdgpu_family }} - run: | - DEST="/opt/rocm-${{ inputs.rocm_version }}" - # Install directly from TheRock release buckets (nightly/dev) using the provided version - python build_tools/install_rocm_from_artifacts.py \ - --release "${{ inputs.rocm_version }}" \ - --artifact-group "${{ inputs.amdgpu_family }}" \ - --output-dir "${DEST}" - # Temporary CI workaround. - # TODO(#4201): Remove this /opt/rocm symlink once JAX wheel - # packaging/RUNPATH fixes are validated and CI no longer depends on - # /opt/rocm -> /opt/rocm- path translation. - ln -sfn "${DEST}" /opt/rocm - - - name: Determine JAX package versions - run: | - # The plugin/pjrt wheels embed the ROCm major version in their package - # name (jax_rocm7_*, jax_rocm10_*), tracking the ROCm release they were - # built against. - rocm_major="${{ inputs.rocm_version }}" - rocm_major="${rocm_major%%.*}" - if [ -z "${rocm_major}" ]; then - echo "::error::rocm_version is required to determine the JAX plugin package name" - exit 1 - fi - echo "JAX_PLUGIN_PACKAGE=jax_rocm${rocm_major}_plugin" >> "$GITHUB_ENV" - echo "JAX_PJRT_PACKAGE=jax_rocm${rocm_major}_pjrt" >> "$GITHUB_ENV" - - # Use versions passed from the build workflow when available. - # For manual workflow_dispatch, compute the version suffix using - # compute_jax_package_version.py (matches rocm-jax ci_build format). - if [ -n "${{ inputs.jax_plugin_version }}" ] && [ -n "${{ inputs.jax_version }}" ]; then - echo "Using versions passed from build workflow" - echo "JAXLIB_VERSION=${{ inputs.jaxlib_version }}" >> "$GITHUB_ENV" - echo "JAX_VERSION=${{ inputs.jax_version }}" >> "$GITHUB_ENV" - echo "JAX_PLUGIN_VERSION=${{ inputs.jax_plugin_version }}" >> "$GITHUB_ENV" - echo "JAX_PJRT_VERSION=${{ inputs.jax_pjrt_version }}" >> "$GITHUB_ENV" - else - echo "Computing versions from rocm_version and rocm-jax requirements" - pip install packaging - python build_tools/github_actions/compute_jax_package_version.py \ - --rocm-version "${{ inputs.rocm_version }}" \ - --jax-requirements rocm-jax/build/requirements.txt - fi - - - name: Install JAX wheels from package index - run: | - echo "Installing JAX packages:" - echo " JAXLIB_VERSION=${JAXLIB_VERSION}" - echo " JAX_PLUGIN_VERSION=${JAX_PLUGIN_VERSION}" - echo " JAX_PJRT_VERSION=${JAX_PJRT_VERSION}" - echo " JAX_VERSION=${JAX_VERSION}" - echo " JAX_PLUGIN_PACKAGE=${JAX_PLUGIN_PACKAGE}" - echo " JAX_PJRT_PACKAGE=${JAX_PJRT_PACKAGE}" - # Install plugin/pjrt from the GPU-family index - pip install --index-url "${{ env.WHEEL_INDEX_URL }}" \ - "${JAX_PLUGIN_PACKAGE}==${JAX_PLUGIN_VERSION}" \ - "${JAX_PJRT_PACKAGE}==${JAX_PJRT_VERSION}" - # For JAX <= 0.9.0, jaxlib is built and installed from GPU index. - # For JAX >= 0.9.1, jaxlib is not built - install jax and jaxlib from upstream PyPI. - if [ -n "${JAXLIB_VERSION}" ]; then - pip install --index-url "${{ env.WHEEL_INDEX_URL }}" "jaxlib==${JAXLIB_VERSION}" - pip install jax==${JAX_VERSION} - else - pip install jax==${JAX_VERSION} jaxlib==${JAX_VERSION} - fi - - - name: Run JAX tests - env: - JAX_PLATFORMS: rocm - HIP_DEVICE_LIB_PATH: /opt/rocm/lib/llvm/amdgcn/bitcode - JAX_ROCM_PLUGIN_INTERNAL_BITCODE_PATH: /opt/rocm/lib/llvm/amdgcn/bitcode - JAX_ROCM_PLUGIN_INTERNAL_LLD_PATH: /opt/rocm/lib/llvm/bin - run: | - pytest jax/tests/multi_device_test.py -q --log-cli-level=INFO - pytest jax/tests/core_test.py -q --log-cli-level=INFO - pytest jax/tests/util_test.py -q --log-cli-level=INFO - pytest jax/tests/scipy_stats_test.py -q --log-cli-level=INFO diff --git a/.github/workflows/test_linux_jax_wheels_partial.yml b/.github/workflows/test_linux_jax_wheels_partial.yml deleted file mode 100644 index 6151ef9d8c..0000000000 --- a/.github/workflows/test_linux_jax_wheels_partial.yml +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test Multi-Arch Linux JAX Wheels - -on: - workflow_call: - inputs: - test_amdgpu_family: - description: GPU family to test (e.g. gfx94X-dcgpu, gfx110X-all) - required: true - type: string - package_index_url: - description: Base URL for the multi-arch Python package index - required: true - type: string - rocm_version: - description: ROCm version - required: true - type: string - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - release_type: - description: "Release type" - type: string - required: true - python_version: - description: Python version to test - required: true - type: string - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag or SHA to checkout." - type: string - default: "" - jax_ref: - description: "rocm-jax repository ref/branch to check out" - required: true - type: string - jax_version: - description: "Built JAX version" - required: false - type: string - jaxlib_version: - description: "Built jaxlib version" - required: false - type: string - jax_plugin_version: - description: "Built jax_rocm7_plugin version" - required: false - type: string - jax_pjrt_version: - description: "Built jax_rocm7_pjrt version" - required: false - type: string - test_runs_on: - description: Runner label to execute tests on - required: true - type: string - workflow_dispatch: - inputs: - test_amdgpu_family: - description: GPU family to test (e.g. gfx94X-dcgpu, gfx110X-all) - type: string - default: "gfx94X-dcgpu" - package_index_url: - description: Base URL for the multi-arch Python package index - type: string - default: "https://rocm.nightlies.amd.com/whl-multi-arch/" - rocm_version: - description: ROCm version - type: string - default: "7.14.0.dev0" - rocm_package_find_links_url: - description: "URL for pip --find-links to install ROCm packages for the JAX manylinux build." - type: string - default: "" - release_type: - description: 'Release type; developer-triggered jobs should use "dev".' - type: choice - options: - - dev - default: dev - python_version: - description: Python version to test - type: string - default: "3.13" - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag or SHA to checkout." - type: string - default: "" - jax_ref: - description: "rocm-jax repository ref/branch to check out" - type: string - default: "rocm-jaxlib-v0.11.0" - jax_version: - description: "Built JAX version" - type: string - default: "" - jaxlib_version: - description: "Built jaxlib version" - type: string - default: "" - jax_plugin_version: - description: "Built jax_rocm7_plugin version" - type: string - default: "" - jax_pjrt_version: - description: "Built jax_rocm7_pjrt version" - type: string - default: "" - test_runs_on: - description: Runner label to use. The selected runner should have a GPU supported by test_amdgpu_family - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" -permissions: - contents: read - id-token: write - -jobs: - test_jax_wheels: - name: Test JAX | ${{ inputs.jax_ref }} | Python ${{ inputs.python_version }} | ${{ inputs.test_amdgpu_family }} - runs-on: ${{ inputs.test_runs_on }} - timeout-minutes: 120 - - container: - image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26 - options: >- - --shm-size=128g - --device /dev/kfd - --device /dev/dri - --group-add render - --group-add video - --user root - --env-file /etc/podinfo/gha-gpu-isolation-settings - --dns 8.8.8.8 - --dns 8.8.4.4 - - defaults: - run: - shell: bash - - env: - VIRTUAL_ENV: ${{ github.workspace }}/.venv - PYTHON_VERSION: ${{ inputs.python_version }} - WHEEL_INDEX_URL: ${{ inputs.package_index_url }} - ROCM_PACKAGE_FIND_LINKS_URL: ${{ inputs.rocm_package_find_links_url }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Checkout rocm-jax - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: rocm-jax - repository: ROCm/rocm-jax - ref: ${{ inputs.jax_ref }} - - - name: Checkout JAX - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: jax - repository: ROCm/jax - ref: ${{ inputs.jax_ref }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - check-latest: true - - - name: System deps, venv configure - run: | - python3 -m venv "${VIRTUAL_ENV}" - echo "PATH=${VIRTUAL_ENV}/bin:${PATH}" >> "$GITHUB_ENV" - python3 build_tools/setup_venv.py \ - "${VIRTUAL_ENV}" \ - --activate-in-future-github-actions-steps - - - name: Install base JAX test requirements - run: | - pip install -r external-builds/jax/requirements-jax.txt - - - name: Determine JAX package versions - run: | - # The plugin/pjrt wheels embed the ROCm major version in their package - # name (jax_rocm7_*, jax_rocm10_*), tracking the ROCm release they were - # built against. - rocm_major="${{ inputs.rocm_version }}" - rocm_major="${rocm_major%%.*}" - if [ -z "${rocm_major}" ]; then - echo "::error::rocm_version is required to determine the JAX plugin package name" - exit 1 - fi - echo "JAX_PLUGIN_PACKAGE=jax_rocm${rocm_major}_plugin" >> "$GITHUB_ENV" - echo "JAX_PJRT_PACKAGE=jax_rocm${rocm_major}_pjrt" >> "$GITHUB_ENV" - - if [ -n "${{ inputs.jax_plugin_version }}" ] && [ -n "${{ inputs.jax_version }}" ]; then - echo "JAX_VERSION=${{ inputs.jax_version }}" >> "$GITHUB_ENV" - echo "JAXLIB_VERSION=${{ inputs.jaxlib_version }}" >> "$GITHUB_ENV" - echo "JAX_PLUGIN_VERSION=${{ inputs.jax_plugin_version }}" >> "$GITHUB_ENV" - echo "JAX_PJRT_VERSION=${{ inputs.jax_pjrt_version }}" >> "$GITHUB_ENV" - else - pip install packaging - - python build_tools/github_actions/compute_jax_package_version.py \ - --rocm-version "${{ inputs.rocm_version }}" \ - --jax-requirements rocm-jax/build/requirements.txt - fi - - name: Compute ROCm device extras - id: expand - run: | - python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.test_amdgpu_family }}" \ - --output-mode=device-extras - - - name: Install ROCm packages - run: | - python -m pip install --upgrade pip - - pkg="rocm[libraries]" - if [ -n "${{ steps.expand.outputs.device_extras }}" ]; then - pkg="${pkg%]},${{ steps.expand.outputs.device_extras }}]" - fi - - python -m pip install --pre \ - --find-links "${ROCM_PACKAGE_FIND_LINKS_URL}" \ - "${pkg}==${{ inputs.rocm_version }}" - - echo "ROCM_ROOT=$(rocm-sdk path --root)" >> "$GITHUB_ENV" - - - name: Install JAX wheels from package index - run: | - echo "Installing from:" - echo " ${WHEEL_INDEX_URL}" - - pip install \ - --index-url "${WHEEL_INDEX_URL}" \ - "${JAX_PLUGIN_PACKAGE}==${JAX_PLUGIN_VERSION}" \ - "${JAX_PJRT_PACKAGE}==${JAX_PJRT_VERSION}" - - if [ -n "${JAXLIB_VERSION}" ]; then - pip install \ - --index-url "${WHEEL_INDEX_URL}" \ - "jaxlib==${JAXLIB_VERSION}" - - pip install "jax==${JAX_VERSION}" - else - pip install \ - "jax==${JAX_VERSION}" \ - "jaxlib==${JAX_VERSION}" - fi - - - name: Run JAX tests - timeout-minutes: 60 - env: - # Avoid large upfront GPU memory reservation. - XLA_PYTHON_CLIENT_PREALLOCATE: "false" - - # Set XLA allocator. JAX 0.10.2 uses the default allocator; all - # other releases keep the platform allocator. - XLA_PYTHON_CLIENT_ALLOCATOR: ${{ inputs.jax_ref == 'rocm-jaxlib-v0.10.2' && 'default' || 'platform' }} - - # JAX 0.10.2 only: ROCm/HIP runtime tuning to avoid a pytest - # slowdown/hang; other releases keep the runtime defaults. - # TODO:(magaonka-amd) remove these as soon as fixes from corresponding system teams lands. - ROCPROFILER_QUEUE_INTERPOSITION: ${{ inputs.jax_ref == 'rocm-jaxlib-v0.10.2' && '0' }} - DEBUG_HIP_DYNAMIC_QUEUES: ${{ inputs.jax_ref == 'rocm-jaxlib-v0.10.2' && '0' }} - HSA_NO_SCRATCH_RECLAIM: ${{ inputs.jax_ref == 'rocm-jaxlib-v0.10.2' && '1' }} - - # Reduce compilation/autotune memory pressure. - XLA_FLAGS: >- - --xla_gpu_force_compilation_parallelism=1 - --xla_gpu_enable_nccl_comm_splitting=false - --xla_gpu_enable_command_buffer= - - run: | - set -euxo pipefail - - unset ROCM_ROOT - unset HIP_DEVICE_LIB_PATH - unset JAX_ROCM_PLUGIN_INTERNAL_BITCODE_PATH - unset JAX_ROCM_PLUGIN_INTERNAL_LLD_PATH - - python -c "import jax; print(jax.local_devices())" - - echo "=== Starting full test suite ===" - pytest jax/tests/multi_device_test.py -q --log-cli-level=INFO - pytest jax/tests/core_test.py -q --log-cli-level=INFO - pytest jax/tests/util_test.py -q --log-cli-level=INFO - pytest jax/tests/scipy_stats_test.py -q --log-cli-level=INFO diff --git a/.github/workflows/test_native_linux_packages_install.yml b/.github/workflows/test_native_linux_packages_install.yml deleted file mode 100644 index e0428b1b1e..0000000000 --- a/.github/workflows/test_native_linux_packages_install.yml +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test Native Linux Packages Install - -on: - workflow_call: - inputs: - package_install_url: - description: "Package installation URL (includes architecture subdirectory for RPM)." - required: true - type: string - release_type: - description: "Release Type (ci, dev, nightly, prerelease)." - required: true - type: string - gpg_key_url: - description: "GPG key URL for signed repos" - required: false - type: string - default: "" - os_profile: - description: OS profile (e.g., ubuntu2404, rhel8, rhel10, sles16). Package type (deb/rpm) is derived from this. - required: true - type: string - artifact_group: - description: Artifact group to extract GPU architecture from (e.g., gfx94X-dcgpu). Default gfx94X-dcgpu. - required: false - type: string - default: "gfx94X-dcgpu" - install_prefix: - description: Installation prefix path (optional, defaults to /opt/rocm/core). - required: false - type: string - default: "/opt/rocm/core" - test_type: - description: "Test type: quick, standard, comprehensive, full, install, or sanity. CI test types are mapped to native install test modes." - required: false - type: string - default: "" - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`" - type: string - required: false - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow" - type: string - required: false - build_variant: - description: "Build variant (e.g. 'asan'). Changes expected package names to match variant-suffixed packages." - required: false - type: string - default: "" - workflow_dispatch: - inputs: - package_install_url: - description: "Package installation URL (includes architecture subdirectory for RPM)" - required: true - type: string - release_type: - description: 'Release type: "ci" or "dev"' - required: true - type: choice - options: - - ci - - dev - gpg_key_url: - description: "GPG key URL for signed repos" - type: string - default: "" - os_profile: - description: OS profile (package type deb/rpm is derived from this). - required: true - type: choice - options: - - ubuntu2404 - - rhel8 - - rhel10 - - sles16 - artifact_group: - description: Artifact group to extract GPU architecture from (e.g., gfx94X-dcgpu) - type: string - default: "gfx94X-dcgpu" - install_prefix: - description: Installation prefix path - type: string - default: "/opt/rocm/core" - required: false - test_type: - description: "Test type: quick, standard, comprehensive, full, install, or sanity (optional; leave empty for sanity)" - type: string - default: "" - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`" - type: string - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow" - type: string - build_variant: - description: "Build variant (e.g. 'asan'). Changes expected package names." - type: string - default: "" - -permissions: - id-token: write - contents: read - -run-name: Test - Native Linux packages Install (${{ inputs.os_profile }}) - -jobs: - prepare_install_context: - name: Prepare install context (package type, GPU arch, container image) - runs-on: ubuntu-24.04 - outputs: - pkg_type: ${{ steps.derive.outputs.pkg_type }} - gfx_arch: ${{ steps.derive.outputs.gfx_arch }} - container_image: ${{ steps.derive.outputs.container_image }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Derive package type, GPU architecture, and container image - id: derive - run: | - case "${{ inputs.os_profile }}" in - ubuntu*|debian*) PKG_TYPE="deb" ;; - *) PKG_TYPE="rpm" ;; - esac - echo "pkg_type=${PKG_TYPE}" >> $GITHUB_OUTPUT - python build_tools/packaging/linux/get_url_repo_params.py extract-gfx-arch \ - --artifact-group "${{ inputs.artifact_group }}" - python build_tools/packaging/linux/get_url_repo_params.py get-container-image \ - --os-profile "${{ inputs.os_profile }}" - - test_native_linux_packages_install: - name: Test Native Linux Packages Installation - ${{ inputs.os_profile }} - needs: [prepare_install_context] - strategy: - fail-fast: false - runs-on: ${{ github.repository_owner == 'ROCm' && 'linux-gfx942-1gpu-ccs-csp-ossci-rocm' || 'ubuntu-24.04' }} - container: - image: ${{ needs.prepare_install_context.outputs.container_image }} - options: --ipc host - env: - REPO_URL: ${{ inputs.package_install_url }} - GPG_KEY_URL: ${{ inputs.gpg_key_url || '' }} - OS_PROFILE: ${{ inputs.os_profile }} - PKG_TYPE: ${{ needs.prepare_install_context.outputs.pkg_type }} - RELEASE_TYPE: ${{ inputs.release_type }} - GFX_ARCH: ${{ needs.prepare_install_context.outputs.gfx_arch }} - INSTALL_PREFIX: ${{ inputs.install_prefix || '/opt/rocm/core' }} - TEST_TYPE: ${{ inputs.test_type }} - BUILD_VARIANT: ${{ inputs.build_variant }} - steps: - - name: Install System Prerequisites - run: | - echo "Installing prerequisites for ${{ env.PKG_TYPE }} on container" - - if [ "${{ env.PKG_TYPE }}" = "deb" ]; then - sudo apt update - sudo apt install -y \ - wget \ - curl \ - ca-certificates \ - gpg \ - lsb-release - if [ "$TEST_TYPE" = "full" ] || [ "$TEST_TYPE" = "comprehensive" ]; then - echo "Extra OS packages for --test-type full/comprehensive (e.g. rdhc): pciutils, kmod, apt-utils" - sudo apt install -y pciutils kmod apt-utils - fi - elif [[ "${{ inputs.os_profile }}" == sles* ]]; then - # SLES prerequisites (uses BCI base - zypper) - zypper refresh - zypper install -y \ - wget \ - curl \ - ca-certificates \ - gpg2 \ - lsb-release \ - update-alternatives - if [ "$TEST_TYPE" = "full" ] || [ "$TEST_TYPE" = "comprehensive" ]; then - echo "Extra OS packages for --test-type full/comprehensive on SLES (e.g. rdhc): pciutils, kmod" - zypper install -y pciutils kmod - fi - else - # rpm (non-SLES on RHEL/UBI); --allowerasing for curl vs curl-minimal on UBI - dnf install -y --allowerasing \ - wget \ - curl \ - ca-certificates \ - gnupg2 \ - dnf-plugins-core - if [ "$TEST_TYPE" = "full" ] || [ "$TEST_TYPE" = "comprehensive" ]; then - echo "Extra OS packages for --test-type full/comprehensive (e.g. rdhc): pciutils, kmod" - dnf install -y --allowerasing pciutils kmod - fi - fi - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Setup Python and install runtime - run: | - bash build_tools/packaging/linux/setup_python_cmd.sh \ - --os-profile "${{ inputs.os_profile }}" \ - --install-runtime \ - --output-format github - - - name: Install Python Dependencies - run: | - # Use venv for all distros for consistency and future-proofing (PEP 668) - $PYTHON_CMD -m venv /tmp/test-venv - /tmp/test-venv/bin/python -m pip install --upgrade pip - /tmp/test-venv/bin/python -m pip install -r build_tools/packaging/linux/tests/requirements.txt - echo "PYTHON_CMD=/tmp/test-venv/bin/python" >> $GITHUB_ENV - - - name: Package Installation Test - run: | - echo "TEST - OS:${{ inputs.os_profile }} REPO:${{ env.REPO_URL }} Arch:${{ env.GFX_ARCH }} Type:${{ env.RELEASE_TYPE }}" - $PYTHON_CMD -m pytest build_tools/packaging/linux/native_linux_package_install_test.py -vv --tb=long -s diff --git a/.github/workflows/test_pytorch_wheels.yml b/.github/workflows/test_pytorch_wheels.yml deleted file mode 100644 index 0cfa0bb245..0000000000 --- a/.github/workflows/test_pytorch_wheels.yml +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test PyTorch Wheels - -on: - workflow_dispatch: - inputs: - amdgpu_family: - description: GPU family to test - required: true - type: string - default: "gfx94X-dcgpu" - test_runs_on: - description: Runner label to use. The selected runner should have a GPU supported by amdgpu_family - required: true - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - package_index_url: - description: Python package index URL to test, typically nightly/dev URL with a "whl-multi-arch" subdir - required: true - type: string - default: "https://rocm.nightlies.amd.com/whl-multi-arch/" - python_version: - required: true - type: string - default: "3.12" - torch_version: - description: torch package version to install. (e.g. "2.7.1+rocm7.10.0a20251120") - required: true - type: string - pytorch_git_ref: - description: PyTorch ref to checkout test sources from. (e.g. "nightly", or "release/2.7") - type: string - default: "release/2.7" - device_query: - description: >- - GPU candidate-set selection (--device-query, stage 1). "unique": one - device per architecture. "all": every device of each architecture. - type: choice - options: - - "unique" - - "all" - default: "unique" - gpu_policy: - description: >- - How many candidate GPUs are made visible (--gpu-policy, stage 2). - "single": one GPU visible at a time. "all": all candidate GPUs visible - at once. - type: choice - options: - - "single" - - "all" - default: "single" - - workflow_call: - inputs: - amdgpu_family: - required: true - type: string - test_runs_on: - required: true - type: string - package_index_url: - required: true - type: string - python_version: - required: true - type: string - torch_version: - required: true - type: string - pytorch_git_ref: - type: string - default: "release/2.7" - device_query: - description: >- - --device-query. "unique": one device per architecture; "all": every - device of each architecture. - type: string - default: "unique" - gpu_policy: - description: >- - --gpu-policy. "single": one GPU visible at a time; "all": all candidate - GPUs visible at once. - type: string - default: "single" - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`." - type: string - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." - type: string - -permissions: - contents: read - -run-name: Test PyTorch (${{ inputs.amdgpu_family }}, ${{ inputs.torch_version}}, ${{ inputs.test_runs_on }}) - -jobs: - test_wheels: - name: Test PyTorch | ${{ inputs.amdgpu_family }} - runs-on: ${{ inputs.test_runs_on }} - container: - image: ${{ contains(inputs.test_runs_on, 'linux') && 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26' || null }} - options: --ipc host - --group-add video - --device /dev/kfd - --device /dev/dri - --group-add 992 - --group-add 110 - --env-file /etc/podinfo/gha-gpu-isolation-settings - --user 0:0 # Running as root, by recommendation of GitHub: https://docs.github.com/en/actions/reference/workflows-and-actions/dockerfile-support#user - defaults: - run: - shell: bash - env: - VENV_DIR: ${{ github.workspace }}/.venv - AMDGPU_FAMILY: ${{ inputs.amdgpu_family }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - # Multi-arch packages use pip extras (e.g. torch[device-gfx942]) to - # select GPU-specific device packages. Expand the family to get the - # comma-separated device extras string for pip install. - - name: Expand GPU family to device extras - id: expand - run: | - python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_family }}" \ - --output-mode=device-extras - - # TODO: also upload and reference test report together with this logging? - - name: Summarize workflow inputs - run: | - python build_tools/github_actions/summarize_test_pytorch_workflow.py \ - --torch-version=${{ inputs.torch_version }} \ - --pytorch-git-ref=${{ inputs.pytorch_git_ref }} \ - --index-url=${{ inputs.package_index_url }} \ - --device-extras="${{ steps.expand.outputs.device_extras }}" - - - name: Set git options - run: | - git config --global core.longpaths true - - # Here we checkout the same version of PyTorch that wheels were built from - # so we have the right set of test source files. We _probably_ don't need - # to run HIPIFY, apply patches, or fetch submodules, so we skip those - # steps to save time. - # TODO(#5110): Accept manifest_url and use checkout_from_manifest.py - # --no-hipify --no-submodules so tests use the exact source commit and - # related repository pins that produced the package under test. - - name: Checkout PyTorch Source Repos from nightly branch - if: ${{ (inputs.pytorch_git_ref == 'nightly') }} - run: | - python external-builds/pytorch/pytorch_torch_repo.py checkout \ - --gitrepo-origin https://github.com/pytorch/pytorch.git \ - --repo-hashtag nightly \ - --no-hipify \ - --no-submodules - - - name: Checkout PyTorch Source Repos from stable branch - if: ${{ (inputs.pytorch_git_ref != 'nightly') }} - run: | - python external-builds/pytorch/pytorch_torch_repo.py checkout \ - --gitrepo-origin https://github.com/ROCm/pytorch.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-hipify \ - --no-submodules - - - name: Set up virtual environment - timeout-minutes: 15 - run: | - python build_tools/setup_venv.py ${VENV_DIR} \ - --packages "torch[${{ steps.expand.outputs.device_extras }}]==${{ inputs.torch_version }}" \ - --index-url=${{ inputs.package_index_url }} \ - --activate-in-future-github-actions-steps - - - name: Install test requirements - run: | - python -m pip install -r external-builds/pytorch/requirements-test.txt - python -m pip install -r external-builds/pytorch/pytorch/.ci/docker/requirements-ci.txt - pip freeze - - - name: Run rocm-sdk sanity tests - timeout-minutes: 5 - run: | - rocm-sdk test - - - name: Run PyTorch smoketests - timeout-minutes: 10 - run: | - python ./external-builds/pytorch/run_pytorch_smoke_tests.py -- \ - --log-cli-level=INFO \ - -v - - - name: Run PyTorch tests - timeout-minutes: 120 - run: | - python ./external-builds/pytorch/run_pytorch_tests.py \ - --device-query "${{ inputs.device_query }}" \ - --gpu-policy "${{ inputs.gpu_policy }}" \ - -- \ - --continue-on-collection-errors \ - --import-mode=importlib \ - -v diff --git a/.github/workflows/test_pytorch_wheels_full.yml b/.github/workflows/test_pytorch_wheels_full.yml deleted file mode 100644 index 412954bfd7..0000000000 --- a/.github/workflows/test_pytorch_wheels_full.yml +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test PyTorch Wheels (Full Suite) - -on: - workflow_dispatch: - inputs: - amdgpu_family: - description: GPU family to test - required: true - type: string - default: "gfx94X-dcgpu" - test_runs_on: - description: Runner label to use. The selected runner should have a GPU supported by amdgpu_family - required: true - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - test_runs_on_multi_gpu: - description: Runner label for multi-GPU test configs (e.g. distributed). Corresponds to test-runs-on-multi-gpu in amdgpu_family_matrix.py - type: string - default: "linux-gfx942-8gpu-ossci-rocm" - package_index_url: - description: Python package index URL to test, typically nightly/dev URL with a "whl-multi-arch" subdir - required: true - type: string - default: "https://rocm.nightlies.amd.com/whl-multi-arch/" - python_version: - required: true - type: string - default: "3.12" - torch_version: - description: torch package version to install. (e.g. "2.7.1+rocm7.10.0a20251120") - required: true - type: string - pytorch_git_ref: - description: PyTorch ref to checkout test sources from. (e.g. "nightly", or "release/2.9") - type: string - default: "nightly" - pytorch_git_repo: - description: "PyTorch git repo to checkout test sources from. (Options: rocm/pytorch, pytorch/pytorch)" - type: string - default: "pytorch/pytorch" - test_configs: - description: Space-separated test configs to run (e.g. "default", "default distributed", "default distributed inductor") - type: string - default: "default distributed inductor" - tests_to_include: - description: Space-separated list of test modules to run (passed as TESTS_TO_INCLUDE to run_test.py). Leave empty to run all tests. - type: string - default: "" - device_query: - description: >- - GPU candidate-set selection (--device-query, stage 1). "unique": one - device per architecture. "all": every device of each architecture. - "auto": derive from the test config ("all" for distributed, "unique" - otherwise). - type: choice - options: - - "auto" - - "unique" - - "all" - default: "auto" - gpu_policy: - description: >- - How many candidate GPUs are made visible (--gpu-policy, stage 2). - "single": one GPU visible at a time. "all": all candidate GPUs visible - at once. "auto": derive from the test config ("all" for distributed, - "single" otherwise). - type: choice - options: - - "auto" - - "single" - - "all" - default: "auto" - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`." - type: string - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." - type: string - - workflow_call: - inputs: - amdgpu_family: - required: true - type: string - test_runs_on: - required: false - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - test_runs_on_multi_gpu: - type: string - default: "linux-gfx942-8gpu-ossci-rocm" - package_index_url: - required: true - type: string - python_version: - required: true - type: string - torch_version: - required: true - type: string - pytorch_git_ref: - type: string - default: "nightly" - pytorch_git_repo: - type: string - default: "pytorch/pytorch" - repository: - description: "Repository to checkout. Otherwise, defaults to `github.repository`." - type: string - ref: - description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." - type: string - test_configs: - type: string - default: "default distributed inductor" - tests_to_include: - description: Space-separated list of test modules to run (passed as TESTS_TO_INCLUDE to run_test.py). Leave empty to run all tests. - type: string - default: "" - device_query: - description: >- - --device-query. "unique": one device per architecture; "all": every - device of each architecture; "auto": derive from the test config. - type: string - default: "auto" - gpu_policy: - description: >- - --gpu-policy. "single": one GPU visible at a time; "all": all candidate - GPUs visible at once; "auto": derive from the test config. - type: string - default: "auto" - -permissions: - contents: read - -run-name: Test PyTorch Full (${{ inputs.amdgpu_family }}, ${{ inputs.torch_version}}, ${{ inputs.test_runs_on }}) - -jobs: - prepare_matrix: - runs-on: ubuntu-24.04 - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - id: generate - run: | - python external-builds/pytorch/generate_test_sharding_matrix.py \ - --test-configs '${{ inputs.test_configs }}' \ - --default-runner '${{ inputs.test_runs_on }}' \ - --multi-gpu-runner '${{ inputs.test_runs_on_multi_gpu }}' - - test_wheels: - needs: prepare_matrix - name: Test PyTorch Full | ${{ inputs.amdgpu_family }} / ${{ matrix.test_config }} (shard ${{ matrix.shard }}/${{ matrix.num_shards }}) - runs-on: ${{ matrix.runs_on }} - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.prepare_matrix.outputs.matrix) }} - container: - image: ${{ contains(inputs.test_runs_on, 'linux') && 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26' || null }} - # SYS_PTRACE is required by PyTorch's test infra which uses gdb to - # collect stack traces on segfaults and hangs (torch/testing/_internal). - options: --shm-size=10g - --cap-add=SYS_PTRACE - --group-add video - --device /dev/kfd - --device /dev/dri - --group-add 992 - --group-add 110 - --env-file /etc/podinfo/gha-gpu-isolation-settings - --user 0:0 # Running as root, by recommendation of GitHub: https://docs.github.com/en/actions/reference/workflows-and-actions/dockerfile-support#user - defaults: - run: - shell: bash - # These env vars are set at the job level so they are available to all steps - # (e.g. export_test_times.py, run_pytorch_tests_full.py). The Python script - # uses os.environ.setdefault() so values set here take precedence. - env: - VENV_DIR: ${{ github.workspace }}/.venv - AMDGPU_FAMILY: ${{ inputs.amdgpu_family }} - SHARD_NUMBER: ${{ matrix.shard }} - NUM_TEST_SHARDS: ${{ matrix.num_shards }} - CI: "1" - PYTORCH_TEST_WITH_ROCM: "1" - PYTORCH_TESTING_DEVICE_ONLY_FOR: "cuda" - # Pin OpenBLAS threads to 1 for deterministic reductions; without this, - # OpenBLAS defaults to the number of CPU cores, which causes - # non-deterministic results in tests like test_entropy_monte_carlo. - OPENBLAS_NUM_THREADS: "1" - PYTORCH_NUM_PYTEST_RERUNS: "2" - TEST_CONFIG: ${{ matrix.test_config }} - TESTS_TO_INCLUDE: ${{ inputs.tests_to_include }} - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - # Multi-arch packages use pip extras (e.g. torch[device-gfx942]) to - # select GPU-specific device packages. Expand the family to get the - # comma-separated device extras string for pip install. - - name: Expand GPU family to device extras - id: expand - run: | - python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_family }}" \ - --output-mode=device-extras - - # safe.directory must be set before Runner Health Status - - name: Adjust git config - run: | - git config --global --add safe.directory $PWD - git config fetch.parallel 10 - git config --global core.longpaths true - - # Here we checkout the same version of PyTorch that wheels were built from - # so we have the right set of test source files. We _probably_ don't need - # to run HIPIFY, apply patches, or fetch submodules, so we skip those - # steps to save time. - - name: Checkout PyTorch Source - run: | - python external-builds/pytorch/pytorch_torch_repo.py checkout \ - --gitrepo-origin https://github.com/${{ inputs.pytorch_git_repo }}.git \ - --repo-hashtag ${{ inputs.pytorch_git_ref }} \ - --no-hipify \ - --no-submodules - - - name: Set up virtual environment - timeout-minutes: 15 - run: | - python build_tools/setup_venv.py ${VENV_DIR} \ - --packages "torch[${{ steps.expand.outputs.device_extras }}]==${{ inputs.torch_version }} rocm[devel,${{ steps.expand.outputs.device_extras }}]" \ - --index-url=${{ inputs.package_index_url }} \ - --activate-in-future-github-actions-steps - - - name: Initialize ROCm SDK and configure environment - run: | - rocm-sdk init - - ROCM_HOME="$(rocm-sdk path --root)" - ROCM_BIN="$(rocm-sdk path --bin)" - ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" - - echo "ROCM_HOME=${ROCM_HOME}" - echo "ROCM_BIN=${ROCM_BIN}" - echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" - - ROCM_SYSDEPS="${ROCM_HOME}/lib/rocm_sysdeps" - ROCM_SYSDEPS_INCLUDE="${ROCM_SYSDEPS}/include" - ROCM_SYSDEPS_PKGCONFIG="${ROCM_SYSDEPS}/lib/pkgconfig" - - # Add ROCm bin to PATH for all subsequent steps - echo "${ROCM_BIN}" >> "${GITHUB_PATH}" - - # Persist ROCm env vars across workflow steps (mirrors install_rocm.sh) - echo "ROCM_PATH=${ROCM_HOME}" >> "${GITHUB_ENV}" - echo "ROCM_HOME=${ROCM_HOME}" >> "${GITHUB_ENV}" - echo "ROCM_SOURCE_DIR=${ROCM_HOME}" >> "${GITHUB_ENV}" - echo "ROCM_BIN=${ROCM_BIN}" >> "${GITHUB_ENV}" - echo "ROCM_CMAKE=${ROCM_CMAKE_PREFIX}" >> "${GITHUB_ENV}" - echo "CMAKE_PREFIX_PATH=${ROCM_CMAKE_PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "${GITHUB_ENV}" - echo "HIP_DEVICE_LIB_PATH=${ROCM_HOME}/lib/llvm/amdgcn/bitcode" >> "${GITHUB_ENV}" - echo "ROCM_DEVICE_LIB_PATH=${ROCM_HOME}/lib/llvm/amdgcn/bitcode" >> "${GITHUB_ENV}" - echo "ROCM_SYSDEPS_INCLUDE=${ROCM_SYSDEPS_INCLUDE}" >> "${GITHUB_ENV}" - echo "CPLUS_INCLUDE_PATH=${ROCM_SYSDEPS_INCLUDE}:${CPLUS_INCLUDE_PATH:-}" >> "${GITHUB_ENV}" - echo "C_INCLUDE_PATH=${ROCM_SYSDEPS_INCLUDE}:${C_INCLUDE_PATH:-}" >> "${GITHUB_ENV}" - echo "PKG_CONFIG_PATH=${ROCM_SYSDEPS_PKGCONFIG}:${PKG_CONFIG_PATH:-}" >> "${GITHUB_ENV}" - echo "LD_LIBRARY_PATH=${ROCM_HOME}/lib/host-math/lib:${ROCM_SYSDEPS}/lib:${LD_LIBRARY_PATH:-}" >> "${GITHUB_ENV}" - echo "LIBRARY_PATH=${ROCM_HOME}/lib/host-math/lib:${ROCM_SYSDEPS}/lib:${LIBRARY_PATH:-}" >> "${GITHUB_ENV}" - echo "USE_MSLK=0" >> "${GITHUB_ENV}" - - # Use ROCm's clang for JIT C++ compilation (torch.utils.cpp_extension) - # instead of requiring gcc/build-essential. - echo "CC=${ROCM_HOME}/lib/llvm/bin/clang" >> "${GITHUB_ENV}" - echo "CXX=${ROCM_HOME}/lib/llvm/bin/clang++" >> "${GITHUB_ENV}" - - # export_test_times.py must run from the PyTorch repo root with - # PYTHONPATH=. because it imports internal PyTorch modules (e.g. - # tools.stats.*) that expect the repo root on sys.path. - - name: Export test times for sharding - run: | - pushd external-builds/pytorch/pytorch - PYTHONPATH=. python tools/stats/export_test_times.py - popd - - - name: Install test requirements - run: | - python -m pip install -r external-builds/pytorch/requirements-test.txt - python -m pip install -r external-builds/pytorch/pytorch/.ci/docker/requirements-ci.txt - pip freeze - - - name: Runner health status - run: | - ./build_tools/health_status.py - - - name: Run rocm-sdk sanity tests - timeout-minutes: 5 - run: | - rocm-sdk test - - - name: Run PyTorch smoketests - timeout-minutes: 10 - run: | - python ./external-builds/pytorch/run_pytorch_smoke_tests.py -- \ - --log-cli-level=INFO \ - -v - - - name: (Linux) Run PyTorch full test suite - if: ${{ runner.os == 'Linux' }} - run: | - python ./external-builds/pytorch/run_pytorch_tests_full.py \ - --device-query "${{ inputs.device_query }}" \ - --gpu-policy "${{ inputs.gpu_policy }}" \ - -- \ - --continue-on-collection-errors - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: test-reports-${{ matrix.test_config }}-${{ matrix.shard }}-${{ matrix.num_shards }} - path: external-builds/pytorch/pytorch/test/test-reports/ - if-no-files-found: ignore - retention-days: 30 - - # HACK: The test process does not terminate on its own gracefully, - # so we write to run_pytorch_tests_exit_code.txt and then kill the process. - # After killing the process we read the return code to signal it normally. - # See https://github.com/ROCm/TheRock/issues/999. - - name: (Windows) Run PyTorch tests - if: ${{ runner.os == 'Windows'}} - continue-on-error: true - run: | - python ./external-builds/pytorch/run_pytorch_tests.py -- \ - --continue-on-collection-errors \ - -v - - - name: (Windows) Read and propagate exit code - if: ${{ runner.os == 'Windows'}} - run: | - if [ -f run_pytorch_tests_exit_code.txt ]; then - EXIT_CODE=$(cat run_pytorch_tests_exit_code.txt) - echo "Exit code from file: ${EXIT_CODE}" - exit ${EXIT_CODE} - else - echo "No run_pytorch_tests_exit_code.txt found" - exit 1 - fi - - # Aggregate every shard's JUnit XML test reports into a single failure table - # on the run's summary page, so all failures across configs/shards are visible - # in one place without digging into individual matrix jobs or artifacts. - test_summary: - if: always() - needs: test_wheels - name: Test Summary - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref || '' }} - sparse-checkout: external-builds/pytorch/summarize_test_results.py - - - name: Download all test report artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: artifacts - pattern: test-reports-* - - - name: Generate combined summary - env: - AMDGPU_FAMILY: ${{ inputs.amdgpu_family }} - TORCH_VERSION: ${{ inputs.torch_version }} - PYTHON_VERSION: ${{ inputs.python_version }} - PACKAGE_INDEX_URL: ${{ inputs.package_index_url }} - PYTORCH_GIT_REF: ${{ inputs.pytorch_git_ref }} - run: | - python external-builds/pytorch/summarize_test_results.py \ - --combined-dir artifacts diff --git a/.github/workflows/test_rocm_wheels.yml b/.github/workflows/test_rocm_wheels.yml deleted file mode 100644 index e4d22b8281..0000000000 --- a/.github/workflows/test_rocm_wheels.yml +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Test ROCm Wheels - -on: - workflow_dispatch: - inputs: - amdgpu_family: - description: GPU family to test (e.g., gfx94X-dcgpu, gfx110X-all) - required: true - type: string - default: "gfx94X-dcgpu" - test_runs_on: - description: Runner label to use. The selected runner should have a GPU supported by amdgpu_family - required: true - type: string - default: "linux-gfx942-1gpu-ccs-csp-ossci-rocm" - package_index_url: - description: >- - Full Python package index URL for use with --index-url. - type: string - default: "https://rocm.nightlies.amd.com/whl-multi-arch/" - package_find_links_url: - description: Full Python package index URL for use with --find-links (usually instead of package_index_url) - type: string - python_version: - required: true - type: string - default: "3.12" - rocm_version: - description: ROCm version to pip install (e.g. "7.10.0a20251124") - required: true - type: string - kpack_split: - description: "'true' if this is a kpack-split flat build." - type: string - default: "false" - container_image_name: - description: >- - Short, human-readable label for the container image (e.g. 'ubuntu24.04', - 'ubi10', 'native'). Use 'native' (with an empty container_image_url) for - native runners like Windows. - type: string - default: "ubuntu24.04" - container_image_url: - description: >- - Container image reference (e.g. 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:...'). - Only applied when test_runs_on targets Linux. Leave empty to run directly - on the runner (required for Windows and native Linux runners). - type: string - default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:405945a40deaff9db90b9839c0f41d4cba4a383c1a7459b28627047bf6302a26" - - workflow_call: - inputs: - amdgpu_family: - required: true - type: string - test_runs_on: - required: true - type: string - package_index_url: - description: >- - Full Python package index URL for use with --index-url. - type: string - package_find_links_url: - type: string - python_version: - required: true - type: string - rocm_version: - required: true - type: string - container_image_name: - description: >- - Short, human-readable label for the container image (e.g. 'ubuntu24.04', - 'ubi10', 'native'). Callers must pass this explicitly so this workflow - stays image-agnostic. Use 'native' (with an empty container_image_url) - for native runners like Windows. - required: true - type: string - container_image_url: - description: >- - Container image reference. Only applied when test_runs_on targets Linux. - Pass an empty string to run directly on the runner (required for Windows - and native Linux runners). - required: true - type: string - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." - type: string - default: "" - -permissions: - contents: read - -run-name: Test ROCm Wheels (${{ inputs.amdgpu_family }}, py${{ inputs.python_version }}, ${{ inputs.rocm_version }}, ${{ inputs.test_runs_on }}) - -jobs: - test_wheels: - name: Test ROCm wheels | ${{ inputs.amdgpu_family }} | py${{ inputs.python_version }} | ${{ inputs.container_image_name }} - if: ${{ inputs.test_runs_on != '' }} - runs-on: ${{ inputs.test_runs_on }} - timeout-minutes: 30 - container: - image: ${{ contains(inputs.test_runs_on, 'linux') && inputs.container_image_url != '' && inputs.container_image_url || null }} - options: --ipc host - --group-add video - --device /dev/kfd - --device /dev/dri - --group-add 992 - --group-add 110 - --env-file /etc/podinfo/gha-gpu-isolation-settings - --user 0:0 # Running as root, by recommendation of GitHub: https://docs.github.com/en/actions/reference/workflows-and-actions/dockerfile-support#user - defaults: - run: - shell: bash - env: - VENV_DIR: ${{ github.workspace }}/.venv - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - # actions/setup-python only has prebuilt binaries for Ubuntu - - name: Set up Python - if: ${{ inputs.container_image_url == '' || contains(inputs.container_image_name, 'ubuntu') }} - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ inputs.python_version }} - - - name: Compute ROCm device extras - id: device_extras - run: | - python build_tools/github_actions/expand_amdgpu_families.py \ - --amdgpu-families "${{ inputs.amdgpu_family }}" \ - --output-mode=device-extras - - # Key off install_libraries success (not test_libraries) so devel install/test still run - - name: Install rocm[libraries] - id: install_libraries - run: | - pkg="rocm[libraries]" - if [ -n "${{ steps.device_extras.outputs.device_extras }}" ]; then - pkg="${pkg%]},${{ steps.device_extras.outputs.device_extras }}]" - fi - python build_tools/setup_venv.py ${VENV_DIR} \ - --packages "${pkg}==${{ inputs.rocm_version }}" \ - --index-url=${{ inputs.package_index_url }} \ - --find-links=${{ inputs.package_find_links_url }} \ - --activate-in-future-github-actions-steps - - - name: Show installed packages (libraries) - if: ${{ !cancelled() && steps.install_libraries.outcome == 'success' }} - run: | - pip freeze - - - name: Test rocm[libraries] - id: test_libraries - if: ${{ !cancelled() && steps.install_libraries.outcome == 'success' }} - timeout-minutes: 5 - run: | - rocm-sdk test - - - name: Install rocm[libraries,devel] - id: install_devel - if: ${{ !cancelled() && steps.install_libraries.outcome == 'success' }} - run: | - pkg="rocm[libraries,devel]" - if [ -n "${{ steps.device_extras.outputs.device_extras }}" ]; then - pkg="${pkg%]},${{ steps.device_extras.outputs.device_extras }}]" - fi - python build_tools/setup_venv.py ${VENV_DIR} \ - --packages "${pkg}==${{ inputs.rocm_version }}" \ - --index-url=${{ inputs.package_index_url }} \ - --find-links=${{ inputs.package_find_links_url }} - - - name: Show installed packages (libraries,devel) - if: ${{ !cancelled() && steps.install_devel.outcome == 'success' }} - run: | - pip freeze - - - name: Test rocm[libraries,devel] - if: ${{ !cancelled() && steps.install_devel.outcome == 'success' }} - timeout-minutes: 5 - run: | - rocm-sdk test diff --git a/.github/workflows/therock-pr-bot.yml b/.github/workflows/therock-pr-bot.yml deleted file mode 100644 index 285657de92..0000000000 --- a/.github/workflows/therock-pr-bot.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: TheRock PR Bot - -on: - workflow_dispatch: - # SECURITY: pull_request_target runs in the BASE-repo context, so secrets and - # a write token are available even for PRs opened from FORKS. - # - # This is SAFE here ONLY because this workflow: - # • NEVER checks out or executes PR head code (see checkout step below), - # • runs the TRUSTED policy_check.py / policy.yml from the base branch, and - # • reads all PR data via the GitHub API. - # Do NOT add build/test/install steps that run PR-provided code, and do NOT - # change the checkout to the PR head — that would create a "pwn request". - # Refs: - # https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target - # https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ - pull_request_target: - branches: [main, pr-bot-test] - types: [opened, synchronize, reopened, ready_for_review, edited] - -# Least privilege. Writes are needed to post the results comment and manage the -# "Not ready to Review" label; everything else stays read-only. -permissions: - contents: read - pull-requests: write # post / update the policy results table comment - issues: write # add / remove the "Not ready to Review" label - checks: read - -# Avoid two runs racing on the same PR's comment/label (e.g. fast pushes). -concurrency: - group: therock-pr-bot-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - policy: - name: therock-pr-bot - runs-on: ubuntu-24.04 - - steps: - # SECURITY: check out the TRUSTED base commit, NEVER the PR head. This - # guarantees we run OUR policy_check.py / policy.yml — not the fork's. - # `persist-credentials: false` follows the safer-checkout defaults - # (https://github.blog/changelog/2026-06-18-safer-pull_request_target-defaults-for-github-actions-checkout/). - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.event.pull_request.base.sha }} - persist-credentials: false - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.11" - - - name: Detect GitHub App secrets - id: app-secrets - shell: bash - env: - APP_ID: ${{ secrets.THEROCK_PR_BOT_APPID }} - APP_KEY: ${{ secrets.THEROCK_PRBOT_KEY }} - run: | - if [[ -n "$APP_ID" && -n "$APP_KEY" ]]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - fi - - - name: Create GitHub App token - id: app-token - if: steps.app-secrets.outputs.available == 'true' - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.THEROCK_PR_BOT_APPID }} - private-key: ${{ secrets.THEROCK_PRBOT_KEY }} - - # SECURITY: only fixed, trusted dependencies — never installed from the PR. - - name: Install deps - run: python -m pip install --upgrade pip requests pyyaml - - - name: Enforce policy - env: - GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }} - HAS_GITHUB_APP_TOKEN: ${{ steps.app-secrets.outputs.available }} - POST_COMMENTS: ${{ steps.app-secrets.outputs.available }} - MUTATE_PR: ${{ steps.app-secrets.outputs.available }} - OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} - PR_NUMBER: ${{ github.event.pull_request.number }} - SHA: ${{ github.event.pull_request.head.sha }} - run: python skills/therock_pr_bot/policy_check.py --timeout-seconds 900 --poll-seconds 15 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml deleted file mode 100644 index 87f1b415d4..0000000000 --- a/.github/workflows/unit_tests.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -name: Unit Tests - -on: - push: - branches: - - main - - release/therock-* - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - unit_tests: - name: "Unit Tests :: ${{ matrix.runs-on}}" - strategy: - fail-fast: false - matrix: - runs-on: [ubuntu-24.04, windows-2022] - runs-on: ${{ matrix.runs-on }} - defaults: - run: - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: Install python deps - run: pip install -r requirements-test.txt - - # Note: build_tools/github_actions/ contains complicated tests in some - # subdirectories that do not support pytest: - # build_tools/github_actions/benchmarks/scripts/ - # build_tools/github_actions/test_executable_scripts/ - # We could move them if we want a simple `pytest build_tools/` to work. - - name: Test build_tools - working-directory: build_tools - run: | - python -m pytest -vv \ - --cov --cov-report=term-missing --cov-report=html - - - name: Test scan_tools - working-directory: build_tools/scan_tools - run: python -m pytest -vv - - # TODO: upload to AWS S3 instead (so reports are hosted and don't require - # a download step to view). - # We could also use https://about.codecov.io/ instead of custom infra. - - name: Upload coverage report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-report-${{ matrix.runs-on }} - path: build/coverage-html/ - - unit_tests_summary: - name: Unit Tests Summary - if: always() - needs: - - unit_tests - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Evaluate workflow results - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python build_tools/github_actions/workflow_summary.py \ - --needs-json '${{ toJSON(needs) }}'