TheRock/rocm-systems/.github/workflows/rccl_perf_regression.yml

# RCCL paired A/B perf-regression gate.
#
# DEPLOYMENT: RCCL now lives in the ROCm/rocm-systems monorepo at projects/rccl.
# This file belongs in that repo at
#   .github/workflows/rccl_perf_regression.yml
# It runs on the self-hosted org runner "tw-gfx950-ainic-ROCm-scale-runner"
# (org: ROCm). Adjust `runs-on` labels to match how that runner was registered.
# The `paths` filter keeps the gate scoped to RCCL-only changes in the monorepo;
# build_rccl.sh auto-detects the projects/rccl cmake subdir in the checkout.
#
# The Slurm submit/poll/report scripts these steps call
# (rccl_build.sbatch, rccl_ab.sbatch, submit_and_poll.sh, format_report.py)
# are version-controlled in ROCm/cvs, on a branch with further related
# changes landing before it merges to main:
#   https://github.com/ROCm/cvs/tree/aimvt-196-rccl-regression-robustness/ci/rccl_perf_gate
#
# Flow per PR:
#   1. Check out PR head with full history (merge-base resolution needs it).
#   2. [build job] rccl_build.sbatch: build librccl.so for reference (merge-base)
#      and candidate (PR head) on a single build node, then template their lib
#      dirs into a per-run copy of ci_detect.json. This is a single-node CPU
#      compile with no RDMA/MPI involved, so it runs under its own concurrency
#      lock instead of the pinned 4-node lock below -- it no longer holds all
#      4 nodes idle for its ~15-20min duration.
#   3. [detect job] rccl_ab.sbatch: run the A/B detector on the pinned 4-node
#      set, using the config the build job produced.
#   4. format_report.py: render verdict to the job summary + PR comment.
#   5. Advisory mode: job fails hard only on infra/polling errors, not on
#      a detected regression yet (see PR description).
name: RCCL Perf Regression Gate

on:
  pull_request:
    branches: [main, develop]
    paths:
      - 'projects/rccl/**'
      - '.github/workflows/rccl_perf_regression.yml'
  # Allow manual dispatch for testing the pipeline before the workflow is on develop.
  # Inputs mirror what a PR run would supply; omit them to use the defaults
  # (A=A detect: ref == cand == current built lib, expects 0 regressions).
  workflow_dispatch:
    inputs:
      config:
        description: 'Config JSON path on TW cluster'
        required: false
        default: '/it-share/rccl-ci/configs/ci_detect_prod.json'
      build_rccl:
        description: 'Build librccl from PR source (1=yes, 0=use fixed lib)'
        required: false
        default: '0'

env:
  RCCL_CI_ROOT: /it-share/rccl-ci
  SLURM_RESERVATION: rccl_ci
  NODELIST: mia1-p01-g[22,26,28,32]

permissions:
  contents: read
  pull-requests: write

jobs:
  build:
    # Forked PRs cannot run this workflow: it builds and MPI-executes the PR's
    # own source on a self-hosted runner, so arbitrary fork code would get
    # arbitrary code execution on our infrastructure. Scope to same-repo PRs
    # only; workflow_dispatch is unaffected (it already requires repo write
    # access to trigger).
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    # Separate lock from the detect job below: build never touches the pinned
    # nodes, so it shouldn't queue behind another PR's in-flight A/B run.
    # (Real cross-PR parallelism from this also needs a 2nd self-hosted runner
    # -- see "Runner capacity" in the PR's known gaps. With one runner, this
    # still shortens how long the 4-node reservation is held per PR.)
    concurrency:
      group: rccl-perf-gate-build
      cancel-in-progress: false
    runs-on: [self-hosted, tw-gfx950-ainic-ROCm-scale-runner]
    timeout-minutes: 60
    outputs:
      config_json: ${{ steps.resolve.outputs.config_json }}
    steps:
      - name: Checkout PR (full history)
        uses: actions/checkout@v4
        with:
          # github.sha/github.ref point at the PR's synthetic merge commit on
          # pull_request events, not the PR head -- use the head SHA directly
          # so candidate == PR head exactly (matters for merge-base diffing).
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
          fetch-depth: 0

      - name: Detect changed paths
        id: paths
        if: github.event_name == 'pull_request'
        run: |
          git diff --name-only origin/${{ github.base_ref }}...HEAD \
            | grep -q '^projects/rccl/' \
            && echo "rccl_changed=1" >> "$GITHUB_OUTPUT" \
            || echo "rccl_changed=0" >> "$GITHUB_OUTPUT"

      - name: Build reference + candidate librccl (single build node)
        id: resolve
        env:
          BUILD_RCCL: ${{ github.event_name == 'pull_request' && steps.paths.outputs.rccl_changed || inputs.build_rccl || '0' }}
          CANDIDATE_SRC: ${{ github.workspace }}
          BASE_REF: origin/${{ github.base_ref || 'develop' }}
          _CONFIG: ${{ inputs.config || '/it-share/rccl-ci/configs/ci_detect_prod.json' }}
        run: |
          if [[ "${BUILD_RCCL}" == "1" ]]; then
            # This job's concurrency lock no longer matches the detect job's
            # lock, so another PR's build can now run while this PR is still
            # waiting for/running its detect step. Templating the shared prod
            # config file in place (the old behavior) would let one PR's build
            # clobber another PR's in-flight detect config -- give every run
            # its own copy instead.
            RUN_CONFIG="${RCCL_CI_ROOT}/configs/ci_detect_run_${{ github.run_id }}.json"
            cp "${_CONFIG}" "${RUN_CONFIG}"
            # Set the output before sbatch runs (not after) so the cleanup-on-
            # failure step below can still find this path if sbatch fails.
            echo "config_json=${RUN_CONFIG}" >> "$GITHUB_OUTPUT"
            sbatch --wait \
              --export="ALL,CANDIDATE_SRC=${CANDIDATE_SRC},BASE_REF=${BASE_REF},CONFIG_JSON=${RUN_CONFIG}" \
              --reservation="${SLURM_RESERVATION}" \
              "${RCCL_CI_ROOT}/cvs/ci/rccl_perf_gate/sbatch/rccl_build.sbatch"
          else
            echo "config_json=${_CONFIG}" >> "$GITHUB_OUTPUT"
          fi

      - name: Clean up per-run config copy on build failure
        # Covers the case the detect job's own cleanup step can't: if sbatch
        # fails after the per-run config copy was made, `detect` never runs
        # (needs: build), so nothing else removes this file from NFS.
        if: failure()
        env:
          _CONFIG: ${{ steps.resolve.outputs.config_json }}
        run: |
          case "${_CONFIG}" in
            "${RCCL_CI_ROOT}"/configs/ci_detect_run_*.json) rm -f "${_CONFIG}" ;;
          esac

  detect:
    needs: build
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    # Serialise: the pinned calibration nodes can host only one A/B job at a
    # time, and thresholds are only valid there. Do NOT cancel in-progress (a
    # running allocation must finish cleanly to release the nodes).
    concurrency:
      group: rccl-perf-gate-pinned-nodes
      cancel-in-progress: false
    runs-on: [self-hosted, tw-gfx950-ainic-ROCm-scale-runner]
    timeout-minutes: 540
    steps:
      - name: Run A/B detector (pinned 4-node allocation)
        id: detect
        # Don't fail here; we still want to post the report below -- the final
        # "Gate" step enforces it based on the captured exit code below, not
        # this step's outcome (continue-on-error reports "failure" the same
        # way for a detected regression and for a polling error, so outcome
        # alone can't tell them apart).
        continue-on-error: true
        env:
          _CONFIG: ${{ needs.build.outputs.config_json }}
        run: |
          set +e
          bash "${RCCL_CI_ROOT}/cvs/ci/rccl_perf_gate/submit_and_poll.sh" "${_CONFIG}"
          code=$?
          set -e
          echo "exit_code=${code}" >> "$GITHUB_OUTPUT"
          exit "${code}"

      - name: Render report
        if: always()
        run: |
          python3 "${RCCL_CI_ROOT}/cvs/ci/rccl_perf_gate/format_report.py" \
            "${RCCL_CI_ROOT}/ab_artifacts/ab_regression_report.json" \
            --no-exit-code \
            --output "${RUNNER_TEMP}/rccl_report.md"
          cat "${RUNNER_TEMP}/rccl_report.md" >> "${GITHUB_STEP_SUMMARY}"

      - name: Post / update PR comment
        if: always() && github.event_name == 'pull_request'
        uses: marocchino/sticky-pull-request-comment@f3bbb8300013c686241004a1c1b5d4045b10bc30  # v2
        with:
          header: rccl-perf-gate
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          path: ${{ runner.temp }}/rccl_report.md

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: rccl-ab-report
          path: |
            /it-share/rccl-ci/ab_artifacts/ab_regression_report.json
            /it-share/rccl-ci/ab_artifacts/ab_derived_thresholds.json
            /it-share/rccl-ci/ab_artifacts/rccl_runs.log

      - name: Clean up per-run config copy
        if: always()
        env:
          _CONFIG: ${{ needs.build.outputs.config_json }}
        run: |
          case "${_CONFIG}" in
            "${RCCL_CI_ROOT}"/configs/ci_detect_run_*.json) rm -f "${_CONFIG}" ;;
          esac

      - name: Gate on detector verdict
        if: always()
        run: |
          # submit_and_poll.sh exit codes: 0 = pass, 1 = regression detected,
          # 2 = submission/polling error (job never reached a terminal state).
          # steps.detect.outcome can't distinguish 1 from 2 -- continue-on-error
          # reports "failure" for any nonzero exit -- so branch on the exit
          # code we captured explicitly instead.
          code="${{ steps.detect.outputs.exit_code }}"
          if [[ "${code}" != "0" && "${code}" != "1" ]]; then
            echo "::error::Detector failed to run (SLURM submission/polling error or timeout, exit ${code}). Report on NFS may be stale — not gating on it."
            exit 2
          fi
          # Advisory mode (see PR description): report the verdict but don't
          # fail the job on a detected regression yet. Infra failures above
          # still fail hard regardless of mode.
          python3 "${RCCL_CI_ROOT}/cvs/ci/rccl_perf_gate/format_report.py" \
            "${RCCL_CI_ROOT}/ab_artifacts/ab_regression_report.json" \
            --no-exit-code \
            --output /dev/null