name: rocprofiler-systems CI / Sanitizers

on:
  workflow_call:

env:
  ROCPROFSYS_CI: ON
  ROCPROFSYS_TMPDIR: "%env{PWD}%/testing-tmp"
  SANITIZER_EXIT_CODE: 1

jobs:
  ubuntu-noble-sanitizers:
    name: Sanitizer / ${{ matrix.sanitizer-type }}
    runs-on: ubuntu-latest
    container:
      image: dgaliffiamd/rocprofiler-systems:ci-rocm-7.2-ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        compiler: ["g++"]
        build-dyninst: ["ON"]
        sanitizer-type: ["address", "undefined", "thread"]
    env:
      DEBIAN_FRONTEND: noninteractive
      OMPI_ALLOW_RUN_AS_ROOT: 1
      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
      # Disable OpenMPI's binary patcher - it conflicts with ASAN's
      # mprotect interceptor, causing SIGSEGV in all MPI/UCX tests.
      OMPI_MCA_patcher: ""
      CCACHE_DIR: ${{ github.workspace }}/.ccache

    steps:
      - name: Install git
        run: |
          apt-get update
          apt-get install -y --no-install-recommends git ca-certificates
          git --version

      - uses: actions/checkout@v6
        with:
          sparse-checkout: |
            projects/rocprofiler-systems/
            .gitmodules
          set-safe-directory: true
          submodules: true

      - name: Install Packages
        timeout-minutes: 25
        uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
        with:
          retry_wait_seconds: 30
          timeout_minutes: 25
          max_attempts: 2
          command: |
            apt-get update &&
            apt-get install -y autoconf autotools-dev bash-completion binutils-dev bison \
              build-essential bzip2 chrpath cmake curl environment-modules flex gettext \
              git-core gnupg2 gzip iproute2 libboost-all-dev libdw-dev libdwarf-dev \
              libelf-dev libgmock-dev libgtest-dev libiberty-dev libpapi-dev libpfm4-dev \
              libsqlite3-dev libtbb-dev libtool locales lsb-release m4 ninja-build \
              nlohmann-json3-dev python3-pip software-properties-common unzip wget \
              vim zip zlib1g-dev libucx-dev ucx-utils libomp-dev libopenmpi-dev &&
            apt-get autoclean

      - name: Restore ccache
        uses: actions/cache@v6
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-ubuntu-noble-sanitizers-${{ github.job }}-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-${{ github.sha }}
          restore-keys: |
            ccache-ubuntu-noble-sanitizers-${{ github.job }}-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-

      - name: Configure ccache
        run: |
          mkdir -p "${CCACHE_DIR}"
          ccache --max-size=1G
          ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime,pch_defines
          ccache -z
          ccache -p | head -20

      - name: Configure git safe directory
        run: git config --global --add safe.directory '*'

      - name: Initialize Submodules
        shell: bash
        run: |
          git submodule update --init \
            projects/rocprofiler-systems/external/dyninst \
            projects/rocprofiler-systems/external/timemory

      - name: Find Sanitizer Libraries and Configure Sanitizer Environment
        shell: bash
        working-directory: projects/rocprofiler-systems/
        run: |
          SUPP_DIR=${PWD}/scripts
          SANITIZER_TYPE="${{ matrix.sanitizer-type }}"
          # Sanitizer reports go here as <prefix>.<pid> files. Surfaced by the
          # "Print Sanitizer Logs" step below.
          SANITIZER_LOG_DIR=${PWD}/sanitizer-logs
          mkdir -p "${SANITIZER_LOG_DIR}"
          echo "SANITIZER_LOG_DIR=${SANITIZER_LOG_DIR}" >> $GITHUB_ENV
          # Default test set covers Dyninst rewrite/runtime-instrument families
          # plus avail and unit tests.
          TEST_REGEX='^(rewrite-caller|code-coverage|trace-time-window|rocprofiler-systems-avail-all|rocprof-sys-unit-tests)'
          if [ "${SANITIZER_TYPE}" = "address" ]; then
              ASAN_LIB=$(${{ matrix.compiler }} -print-file-name=libasan.so)
              ASAN_LIB=$(realpath "${ASAN_LIB}")
              echo "ASan library: ${ASAN_LIB}"
              ls -la "${ASAN_LIB}"
              echo "ASAN_LIBRARY=${ASAN_LIB}" >> $GITHUB_ENV
              echo "SANITIZER_CMAKE_FLAGS=-DASAN_LIBRARY=${ASAN_LIB}" >> $GITHUB_ENV
              echo "ASAN_OPTIONS=suppressions=${SUPP_DIR}/rocprof-sys-asan.supp:detect_leaks=1:new_delete_type_mismatch=0:detect_odr_violation=0:exitcode=${SANITIZER_EXIT_CODE}:log_path=${SANITIZER_LOG_DIR}/asan" >> $GITHUB_ENV
              echo "LSAN_OPTIONS=suppressions=${SUPP_DIR}/rocprof-sys-lsan.supp:print_suppressions=1:exitcode=${SANITIZER_EXIT_CODE}:log_path=${SANITIZER_LOG_DIR}/lsan" >> $GITHUB_ENV
              echo "UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=0:exitcode=${SANITIZER_EXIT_CODE}:log_path=${SANITIZER_LOG_DIR}/ubsan" >> $GITHUB_ENV
          elif [ "${SANITIZER_TYPE}" = "thread" ]; then
              TSAN_LIB=$(${{ matrix.compiler }} -print-file-name=libtsan.so)
              TSAN_LIB=$(realpath "${TSAN_LIB}")
              echo "TSan library: ${TSAN_LIB}"
              ls -la "${TSAN_LIB}"
              echo "TSAN_LIBRARY=${TSAN_LIB}" >> $GITHUB_ENV
              echo "SANITIZER_CMAKE_FLAGS=-DTSAN_LIBRARY=${TSAN_LIB}" >> $GITHUB_ENV
              echo "TSAN_OPTIONS=suppressions=${SUPP_DIR}/rocprof-sys-thread.supp:second_deadlock_stack=1:print_suppressions=1:exitcode=${SANITIZER_EXIT_CODE}:log_path=${SANITIZER_LOG_DIR}/tsan" >> $GITHUB_ENV

              # Known TSan issue: logger fork tests (pthread_atfork + fork from
              # multi-threaded process) cause the test binary to hang during
              # process exit. TSan's exit-time cleanup deadlocks with the
              # atfork handlers that reference static locals being destroyed.
              echo "GTEST_FILTER=-logger_test*" >> $GITHUB_ENV

              # TSan's 5-15x slowdown blows past the per-test 300s timeout for
              # the Dyninst rewrite/runtime-instrument families. Restrict the
              # TSan job to the lightweight avail + unit-test families.
              TEST_REGEX='^(rocprofiler-systems-avail-all|rocprof-sys-unit-tests)'
          elif [ "${SANITIZER_TYPE}" = "undefined" ]; then
              echo "SANITIZER_CMAKE_FLAGS=" >> $GITHUB_ENV
              echo "UBSAN_OPTIONS=suppressions=${SUPP_DIR}/rocprof-sys-ubsan.supp:print_stacktrace=1:halt_on_error=0:exitcode=${SANITIZER_EXIT_CODE}:log_path=${SANITIZER_LOG_DIR}/ubsan" >> $GITHUB_ENV
          fi
          echo "TEST_REGEX=${TEST_REGEX}" >> $GITHUB_ENV

      - name: Generate CI scripts
        timeout-minutes: 5
        shell: bash
        working-directory: projects/rocprofiler-systems/
        env:
          GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
          CMAKE_C_COMPILER_LAUNCHER: ccache
          CMAKE_CXX_COMPILER_LAUNCHER: ccache
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
          git config --global --add safe.directory ${PWD}
          echo "CMake: $(cmake --version | head -n 1)"
          echo "Compiler: $(${{ matrix.compiler }} --version | head -n 1)"
          echo "Sanitizer type: ${{ matrix.sanitizer-type }}"
          echo "Sanitizer CMake flags: ${SANITIZER_CMAKE_FLAGS}"
          python3 ./scripts/run-ci.py --stage generate \
            --name "${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-noble-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-sanitizer" \
            --build-jobs $(nproc) \
            --site GitHub \
            -B build \
            -- \
            -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') \
            -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-systems \
            -DROCPROFSYS_BUILD_EXAMPLES=ON \
            -DROCPROFSYS_BUILD_TESTING=ON \
            -DROCPROFSYS_USE_PYTHON=ON \
            -DROCPROFSYS_PYTHON_PREFIX=/opt/conda/envs \
            -DROCPROFSYS_PYTHON_ENVS="py3.8;py3.9;py3.10;py3.11;py3.12;py3.13" \
            -DROCPROFSYS_USE_MPI=OFF \
            -DROCPROFSYS_USE_MPI_HEADERS=OFF \
            -DROCPROFSYS_USE_COMPILE_TIMING=OFF \
            -DROCPROFSYS_USE_SANITIZER=ON \
            -DROCPROFSYS_SANITIZER_TYPE=${{ matrix.sanitizer-type }} \
            ${SANITIZER_CMAKE_FLAGS} \
            -DROCPROFSYS_BUILD_DYNINST=${{ matrix.build-dyninst }} \
            -DROCPROFSYS_BUILD_BOOST=OFF \
            -DROCPROFSYS_BUILD_TBB=OFF \
            -DROCPROFSYS_BUILD_ELFUTILS=OFF \
            -DROCPROFSYS_BUILD_LIBIBERTY=OFF \
            -DROCPROFSYS_MAX_THREADS=64 \
            -DROCPROFSYS_DISABLE_EXAMPLES="transpose;parallel-overhead;user-api;openmp;mpi;python;lulesh;rccl;causal;videodecode;jpegdecode;roctx;thread-limit;transferBench;hpc;shmem;scratch-memory;sdma_test" \
            -DROCPROFSYS_BUILD_NUMBER=${{ github.run_attempt }} \
            -DCMAKE_PREFIX_PATH=/opt/rocm \
            -- \
            -R "${TEST_REGEX}"

      - name: Configure
        timeout-minutes: 15
        shell: bash
        working-directory: projects/rocprofiler-systems/
        env:
          GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
          CMAKE_C_COMPILER_LAUNCHER: ccache
          CMAKE_CXX_COMPILER_LAUNCHER: ccache
        run: |
          python3 ./scripts/run-ci.py --stage configure \
            --name "${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-noble-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-sanitizer" \
            -B build

      - name: Build
        timeout-minutes: 120
        shell: bash
        working-directory: projects/rocprofiler-systems/
        env:
          CMAKE_C_COMPILER_LAUNCHER: ccache
          CMAKE_CXX_COMPILER_LAUNCHER: ccache
        run: |
          python3 ./scripts/run-ci.py --stage build \
            --name "${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-noble-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-sanitizer" \
            -B build

      - name: CDash link
        if: always()
        shell: bash
        run: |
          NAME="${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-noble-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-sanitizer"
          NAME=$(echo "${NAME}" | sed 's/\(.*\)-\([0-9]*\)\/merge/PR_\2_\1/')
          URL="https://my.cdash.org/index.php?project=rocprofiler-systems&filtercount=1&showfilters=1&field1=buildname&compare1=65&value1=${NAME}"
          printf '### CDash Results\n[View build on CDash](%s)\n' "${URL}" >> "${GITHUB_STEP_SUMMARY}"
          echo "::notice title=CDash Results::${URL}"

      - name: Test
        timeout-minutes: 180
        shell: bash
        working-directory: projects/rocprofiler-systems/
        run: |
          python3 ./scripts/run-ci.py --stage test \
            --name "${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-noble-${{ matrix.compiler }}-${{ matrix.sanitizer-type }}-sanitizer" \
            --repeat-until-pass 1 \
            --repeat-after-timeout 1 \
            -B build

      - name: ccache stats
        if: always()
        run: |
          echo "=== ccache stats ==="
          ccache -s -v || true
          echo "=== ccache dir size ==="
          du -sh "${CCACHE_DIR}" || true

      - name: Print Sanitizer Logs
        if: always()
        shell: bash
        working-directory: projects/rocprofiler-systems/
        run: |
          if [ -z "${SANITIZER_LOG_DIR}" ] || [ ! -d "${SANITIZER_LOG_DIR}" ]; then
            echo "No sanitizer log directory at '${SANITIZER_LOG_DIR}'"
            exit 0
          fi
          shopt -s nullglob
          files=("${SANITIZER_LOG_DIR}"/*)
          if [ "${#files[@]}" -eq 0 ]; then
            echo "No sanitizer log files in ${SANITIZER_LOG_DIR}"
            exit 0
          fi
          # Lines that mean a sanitizer actually fired (vs. a benign
          # "Suppressions used:" summary that print_suppressions=1 always emits).
          finding_regex='==[0-9]+==ERROR|SUMMARY: (Address|Leak|Undefined|Thread)Sanitizer|WARNING: ThreadSanitizer|runtime error:'
          files_with_findings=()
          for f in $(printf '%s\n' "${files[@]}" | sort); do
            name=$(basename "${f}")
            if grep -qE "${finding_regex}" "${f}"; then
              files_with_findings+=("${name}")
              echo "::group::${name} (FINDINGS)"
              cat "${f}"
              echo "::endgroup::"
            else
              # Suppression summaries only — collapse without flagging.
              echo "::group::${name} (clean — suppressions only)"
              cat "${f}"
              echo "::endgroup::"
            fi
          done
          if [ "${#files_with_findings[@]}" -gt 0 ]; then
            echo "::error::Sanitizer findings detected in ${#files_with_findings[@]} log file(s): ${files_with_findings[*]}"
            exit 1
          fi
          echo "No sanitizer findings detected (only suppression summaries)."

      - name: Check for Leftover Buffered Files
        timeout-minutes: 5
        working-directory: projects/rocprofiler-systems/
        run: |
          set -v
          if find /tmp -maxdepth 1 -name 'buffered*' -print -quit | grep -q .; then
            echo "Error: Found leftover buffered storage files in /tmp:"
            ls -lh /tmp/buffered*
            exit 1
          else
            echo "✓ No buffered storage files found in /tmp"
          fi

      - name: Upload JUnit test results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: junit-sanitizer-${{ matrix.sanitizer-type }}-${{ github.job }}-${{ strategy.job-index }}
          path: projects/rocprofiler-systems/build/test-results.xml
          if-no-files-found: ignore

      - name: Write sanitizer summary status
        if: always()
        shell: bash
        run: |
          mkdir -p sanitizer-summary
          printf '{"sanitizer":"%s","status":"%s"}\n' '${{ matrix.sanitizer-type }}' '${{ job.status }}' > sanitizer-summary/${{ matrix.sanitizer-type }}.json

      - name: Upload sanitizer summary status
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: sanitizer-summary-${{ matrix.sanitizer-type }}-${{ github.job }}-${{ strategy.job-index }}
          path: sanitizer-summary/${{ matrix.sanitizer-type }}.json
          if-no-files-found: error

      - name: Test Clean Up
        timeout-minutes: 10
        working-directory: projects/rocprofiler-systems/
        run: |
          set -v
          du /tmp -d 1 -h
          du build/rocprof-sys-pytest-output -d 1 -h
          df -h
          rm -fr /tmp/* build/rocprof-sys-pytest-output/*

      - name: CTest Artifacts
        if: failure()
        continue-on-error: True
        uses: actions/upload-artifact@v7
        with:
          name: ctest-${{ matrix.sanitizer-type }}-${{ github.job }}-${{ strategy.job-index }}-log
          path: |
            projects/rocprofiler-systems/build/*.log

      - name: Data Artifacts
        if: failure()
        continue-on-error: True
        uses: actions/upload-artifact@v7
        with:
          name: data-${{ matrix.sanitizer-type }}-${{ github.job }}-${{ strategy.job-index }}-files
          path: |
            projects/rocprofiler-systems/build/rocprofsys-tests-config/*.cfg
            projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*.txt
            projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*-instr*.json