TheRock/rocm-libraries/projects/hipblaslt/tensilelite/tox.ini

# Copyright © Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier: MIT

[tox]
envlist = py3,lint
labels =
    static = format, isort
    coverage = coverage

[testenv]
changedir = {toxinidir}
# Some versions of Pytest versions have a bug:
# https://github.com/pytest-dev/pytest/issues/5971 which causes the whole test
# process to crash if a multiprocessing job has an exception. Fixed in 5.3.3.
deps =
    cmake
    nanobind
    scikit-build-core>=0.10
    setuptools
    -r{toxinidir}/requirements.txt
    pytest>=5.4.1
    pytest-xdist>=1.32.0
    syrupy
    filelock
    invoke
setenv =
    ROCM_PATH = {env:ROCM_PATH:/opt/rocm}
    LD_LIBRARY_PATH = {env:ROCM_PATH:/opt/rocm}/lib:{env:LD_LIBRARY_PATH:}
    TENSILE_CLIENT_STATIC = {env:TENSILE_CLIENT_STATIC:}
    TENSILELITE_CLIENT_ARGS = {env:TENSILELITE_CLIENT_ARGS:}
    TENSILE_NUM_PYTEST_WORKERS = {env:TENSILE_NUM_PYTEST_WORKERS:4}
passenv =
    ; FFM variables
    HSA_MODEL_TOPOLOGY
    HSA_MODEL_TOML
    HSA_MODEL_LIB
    HSA_MODEL_NUM_THREADS
    HSA_MODEL_ARGS
    HSA_MODEL_MEMFILE
    ; These might need to be set when using FFM
    HSA_XNACK
    HSA_ENABLE_SDMA
    HSA_ENABLE_INTERRUPT
    HSA_XNACK
    LD_LIBRARY_PATH
commands =
    pip install --upgrade pip
    pip install {toxinidir}/rocisa/
    pytest -v --basetemp={envtmpdir} --junit-xml={toxinidir}/rocisa_tests.xml --junit-prefix=rocisa --color=yes -n {env:TENSILE_NUM_PYTEST_WORKERS:4} {toxinidir}/rocisa/test
    pip install pytest-cov
    sh -c "ARCH=$(invoke get-gpu-arch); \
          export TENSILE_ARCHITECTURE=\$ARCH; \
          export TENSILELITE_CLIENT_ARGS='{env:TENSILELITE_CLIENT_ARGS:} --gpu-targets '$ARCH; \
          invoke build-client --build-dir {toxinidir}/build_tmp $TENSILELITE_CLIENT_ARGS || exit $?; \
          pytest -v --basetemp={envtmpdir} --junit-xml={toxinidir}/python_tests.xml --junit-prefix={envname} --color=yes -n $TENSILE_NUM_PYTEST_WORKERS --prebuilt-client={toxinidir}/build_tmp/tensilelite/client/tensilelite-client --global-parameters CheckASMCodeSize=True {toxinidir}/Tensile/Tests {posargs}; \
          rc=$?; test $rc -eq 5 && exit 0; exit $rc"
allowlist_externals =
    mkdir
    sh
    cmake

[testenv:py3]
basepython = python3

[testenv:unit]
description = "Runs Python unit tests quickly, skipping the client build. Assumes a build has run before."
basepython = python3
# This environment inherits 'deps' and 'setenv' from [testenv]
commands =
    pip install {toxinidir}/rocisa/
    pytest -v --basetemp={envtmpdir} {posargs}


[testenv:mutation-unit]
description = "Mutation testing (mutmut) for the slice configured in [tool.mutmut] in pyproject.toml. Separate from the normal unit target (see the Mutation testing section of Tensile/Tests/unit/characterization/README.md). Assumes rocisa is built/installed. Report-only on the pilot slice."
basepython = python3
# Inherits 'deps' and 'setenv' from [testenv]; adds the mutation engine.
deps =
    {[testenv]deps}
    mutmut>=3.6
    pytest-cov
commands =
    pip install {toxinidir}/rocisa/
    mutmut run {posargs}
    mutmut results


[testenv:rocisa]
description = "Runs only rocisa tests"
basepython = python3
commands =
    pip install --upgrade pip
    pip install --no-build-isolation {toxinidir}/rocisa/
    pytest -v --basetemp={envtmpdir} --junit-xml={toxinidir}/rocisa_tests.xml --junit-prefix=rocisa --color=yes -n 4 {toxinidir}/rocisa/test {posargs}

[testenv:lint]
basepython = python3
deps =
    -r{toxinidir}/requirements.txt
    joblib
    flake8
commands = flake8 {toxinidir}/Tensile

[testenv:format]
description = "Formats code so developers don't have to"
skip_install = true
deps = black==24.4
commands =
    black \
      --line-length=100 \
      {toxinidir}/Tensile/Common \
      {toxinidir}/Tensile/TensileCreateLibrary \
      {toxinidir}/Tensile/Utilities/Decorators \
      {posargs}

[testenv:isort]
description = "Sorts import statements for less merge conflicts"
skip_install = true
deps = isort==5.13.2
commands =
    isort \
      --profile=black \
      {toxinidir}/Tensile/Common \
      {toxinidir}/Tensile/TensileCreateLibrary \
      {toxinidir}/Tensile/Utilities/Decorators \
      {posargs}

[testenv:pre_commit]
basepython = python3
deps =
    -r{toxinidir}/requirements.txt
    joblib
    pytest>=5.4.1
    syrupy
    flake8
commands =
    flake8 {toxinidir}/Tensile
    py.test -q --basetemp={envtmpdir} --color=yes {toxinidir}/Tensile -m unit {posargs}

[testenv:coverage]
description = "Generate comprehensive code coverage for unit and common tests"
basepython = python3
deps =
    {[testenv]deps}
    pytest-cov>=4.0.0
    coverage[toml]>=7.0.0
    scikit-build-core==0.10.0
setenv =
    {[testenv]setenv}
    # PYTHONPATH is required for coverage.py multiprocessing mode to track source in pytest-xdist workers
    PYTHONPATH = {toxinidir}
    COVERAGE_FILE = {toxinidir}/.coverage
commands =
    # Install rocisa
    pip install --upgrade pip
    pip install {toxinidir}/rocisa/

    # Build the client
    sh -c "ARCH=$(invoke get-gpu-arch); \
          export TENSILE_ARCHITECTURE=\$ARCH; \
          export TENSILELITE_CLIENT_ARGS='{env:TENSILELITE_CLIENT_ARGS:} --gpu-targets '$ARCH; \
          invoke build-client --build-dir {toxinidir}/build_tmp $TENSILELITE_CLIENT_ARGS"

    # Run unit tests with coverage (no report yet)
    pytest --cov=Tensile --cov=rocisa \
           --cov-config={toxinidir}/pyproject.toml \
           --cov-report= \
           -v --basetemp={envtmpdir} \
           --junit-xml={toxinidir}/unit_tests.xml \
           --junit-prefix=unit \
           --color=yes \
           -n {env:TENSILE_NUM_PYTEST_WORKERS:4} \
           -m unit \
           {toxinidir}/Tensile/Tests/unit

    # Run common tests with coverage (append to existing coverage data)
    pytest --cov=Tensile --cov=rocisa \
           --cov-config={toxinidir}/pyproject.toml \
           --cov-append \
           --cov-report=html:{toxinidir}/htmlcov \
           --cov-report=xml:{toxinidir}/coverage.xml \
           --cov-report=json:{toxinidir}/coverage.json \
           --cov-report=term-missing:skip-covered \
           -v --basetemp={envtmpdir} \
           --junit-xml={toxinidir}/common_tests.xml \
           --junit-prefix=common \
           --color=yes \
           -n {env:TENSILE_NUM_PYTEST_WORKERS:4} \
           -m common \
           --prebuilt-client={toxinidir}/build_tmp/tensilelite/client/tensilelite-client \
           --global-parameters CheckASMCodeSize=True \
           {toxinidir}/Tensile/Tests/common {posargs}

    # Generate detailed coverage report summary
    coverage report --show-missing --skip-covered

    # Print report locations
    python -c "print('\n' + '='*70); \
               print('CODE COVERAGE REPORTS GENERATED:'); \
               print('='*70); \
               print('HTML Report : htmlcov/index.html'); \
               print('XML Report  : coverage.xml'); \
               print('JSON Report : coverage.json'); \
               print('JUnit XML   : unit_tests.xml, common_tests.xml'); \
               print('='*70 + '\n')"

[testenv:coverage-cpp]
description = "Generate C++ code coverage for tensilelite-host library"
basepython = python3
deps =
    {[testenv]deps}
    lcov_cobertura
    pycobertura
setenv =
    {[testenv]setenv}
    LLVM_PROFILE_FILE = {toxinidir}/build_cov/profraw/cpp_test_%p_%m.profraw
    LD_LIBRARY_PATH = {env:LD_LIBRARY_PATH:}:{env:ROCM_PATH:/opt/rocm}/lib/llvm/lib
passenv =
    {[testenv]passenv}
allowlist_externals =
    {[testenv]allowlist_externals}
    ctest
    lcov
    genhtml
commands =

    # Build with coverage instrumentation
    sh -c "ARCH=$(invoke get-gpu-arch); \
          invoke build-coverage --build-dir {toxinidir}/build_cov --gpu-targets $ARCH"

    # Create directories
    sh -c "mkdir -p {toxinidir}/build_cov/profraw {toxinidir}/coverage_cpp"

    # Run C++ gtest tests (tensilelite-tests) for tensilelite-host coverage
    # Excluded Tests: Pre-existing failures on gfx950
    # - RangeLibraryTest MI350 gfx950: Assertion failure
    # - FP4ZeroFrequency/MXDataGenFP4Test.ZeroFrequencyWithinBounds/(2048, 514, 32, false): SEGFAULT
    # TODO: <AIHPBLAS-3506> Investigate and fix these test failures
    sh -c "export LLVM_PROFILE_FILE='{toxinidir}/build_cov/profraw/cpp_test_%p_%m.profraw'; \
          cd {toxinidir}/build_cov/tensilelite/tests && \
          ctest --output-on-failure --timeout 120 -E '(RangeLibraryTest.*MI350.*gfx950|FP4ZeroFrequency.*2048.*514.*32.*false)'; \
          echo $? > {toxinidir}/.cpp_test_exit_code"

    # Run rocisa Python tests to get coverage for rocisa C++ code
    # Excluded Test: crashes with SIGABRT in coverage-instrumented rocisa builds.
    # Assertion failure in rocisa/rocisa/src/pass/macro_inline.cpp:315 expandMacroBody().
    # TODO::Fix and remove the --deselect flag below.
    sh -c "export LLVM_PROFILE_FILE='{toxinidir}/build_cov/profraw/rocisa_%p_%m.profraw'; \
          cd {toxinidir} && \
          PYTHONPATH='{toxinidir}/build_cov/tensilelite/rocisa' python -m pytest rocisa/test -v --tb=short \
          --deselect=rocisa/test/test_macro_inline.py::test_prnd_generator \
          2>&1; \
          echo $? > {toxinidir}/.rocisa_test_exit_code"

    # Generate C++ coverage report using LLVM tools and lcov
    # Note: Excludes .hpp headers only (coverage-cpp focuses on .cpp files).
    # Gap: header-implemented code in .hpp files (templates, inline functions) won't appear in coverage.
    sh -c "PROFDATA=$(which llvm-profdata); \
          LLVM_COV=$(which llvm-cov); \
          OUT={toxinidir}/coverage_cpp; \
          \
          echo 'Merging profraw files...'; \
          $PROFDATA merge -sparse -o $OUT/cpp.profdata {toxinidir}/build_cov/profraw/*.profraw; \
          \
          echo 'Exporting to LCOV format...'; \
          ROCISA_SO=$(find {toxinidir}/build_cov -name '_rocisa*.so' | head -1); \
          $LLVM_COV export -format=lcov -instr-profile=$OUT/cpp.profdata \
              -object {toxinidir}/build_cov/tensilelite/tests/tensilelite-tests \
              -object {toxinidir}/build_cov/tensilelite/client/tensilelite-client \
              -object {toxinidir}/build_cov/tensilelite/client/cpu-gemm-driver \
              -object $ROCISA_SO \
              --ignore-filename-regex='.*/(build|_deps|site-packages)/.*' \
              --ignore-filename-regex='.*/origami/.*' \
              --ignore-filename-regex='.*/stinkytofu/.*' \
              --ignore-filename-regex='.*/tests/.*' \
              --ignore-filename-regex='.*_test\\.cpp$' \
              --ignore-filename-regex='.*\\.hpp$' \
              --ignore-filename-regex='.*/library/include/.*' \
              > $OUT/combined.lcov; \
          \
          echo 'Generating HTML report...'; \
          genhtml $OUT/combined.lcov -o $OUT/html --title 'TensileLite C++ Coverage' \
              --legend --ignore-errors source -p {toxinidir} 2>/dev/null || true; \
          \
          echo 'Generating Cobertura XML...'; \
          lcov_cobertura $OUT/combined.lcov -o $OUT/coverage.xml 2>/dev/null || true"

    # Print C++ coverage report
    sh -c "echo ''; \
           echo '======================================================================'; \
           echo 'C++ COVERAGE REPORT'; \
           echo '======================================================================'; \
           pycobertura show {toxinidir}/coverage_cpp/coverage.xml 2>/dev/null || \
               echo 'Coverage report not available'; \
           echo ''; \
           echo '======================================================================'; \
           echo 'REPORTS:'; \
           echo '  HTML Report : coverage_cpp/html/index.html'; \
           echo '  LCOV Data   : coverage_cpp/combined.lcov'; \
           echo '  Cobertura   : coverage_cpp/coverage.xml'; \
           echo '======================================================================'"

    # Print test results and check exit codes
    python -c "import sys; \
               import os; \
               cpp_rc = int(open('.cpp_test_exit_code').read().strip()); \
               rocisa_rc = int(open('.rocisa_test_exit_code').read().strip()); \
               print('\\nTest Results:'); \
               print('  C++ tests: exit_code={} - {}'.format(cpp_rc, 'PASSED' if cpp_rc == 0 else 'FAILED')); \
               print('  Rocisa tests: exit_code={} - {}'.format(rocisa_rc, 'PASSED' if rocisa_rc == 0 else 'FAILED')); \
               failed = cpp_rc != 0 or rocisa_rc != 0; \
               print('\\nOverall: {}'.format('FAILED' if failed else 'PASSED')); \
               sys.exit(1 if failed else 0)"

[testenv:coverage-unit]
description = "Generate code coverage for unit tests only (fast)"
basepython = python3
deps =
    {[testenv]deps}
    pytest-cov>=4.0.0
    coverage[toml]>=7.0.0
    scikit-build-core==0.10.0
setenv =
    {[testenv]setenv}
    # PYTHONPATH is required for coverage.py multiprocessing mode to track source in pytest-xdist workers
    PYTHONPATH = {toxinidir}
    COVERAGE_FILE = {toxinidir}/.coverage
commands =
    pip install --upgrade pip
    pip install {toxinidir}/rocisa/

    pytest --cov=Tensile --cov=rocisa \
           --cov-config={toxinidir}/pyproject.toml \
           --cov-report=html:{toxinidir}/htmlcov \
           --cov-report=xml:{toxinidir}/coverage.xml \
           --cov-report=term-missing \
           -v --basetemp={envtmpdir} \
           --junit-xml={toxinidir}/unit_tests.xml \
           --color=yes \
           -n {env:TENSILE_NUM_PYTEST_WORKERS:4} \
           -m unit \
           {toxinidir}/Tensile/Tests/unit {posargs}

commands_post =
    # Always runs, even if tests fail
    coverage report --show-missing

    python -c "print('\nHTML Report: htmlcov/index.html')"

[testenv:coverage-common]
description = "Generate code coverage for common tests only (requires prebuilt client)"
basepython = python3
deps =
    {[testenv]deps}
    pytest-cov>=4.0.0
    coverage[toml]>=7.0.0
    scikit-build-core==0.10.0
setenv =
    {[testenv]setenv}
    # PYTHONPATH is required for coverage.py multiprocessing mode to track source in pytest-xdist workers
    PYTHONPATH = {toxinidir}
    COVERAGE_FILE = {toxinidir}/.coverage
commands =
    pip install --upgrade pip
    pip install {toxinidir}/rocisa/

    sh -c "ARCH=$(invoke get-gpu-arch); \
          export TENSILE_ARCHITECTURE=\$ARCH; \
          export TENSILELITE_CLIENT_ARGS='{env:TENSILELITE_CLIENT_ARGS:} --gpu-targets '$ARCH; \
          invoke build-client --build-dir {toxinidir}/build_tmp $TENSILELITE_CLIENT_ARGS"

    pytest --cov=Tensile --cov=rocisa \
           --cov-config={toxinidir}/pyproject.toml \
           --cov-report=html:{toxinidir}/htmlcov \
           --cov-report=xml:{toxinidir}/coverage.xml \
           --cov-report=term-missing \
           -v --basetemp={envtmpdir} \
           --junit-xml={toxinidir}/common_tests.xml \
           --color=yes \
           -n {env:TENSILE_NUM_PYTEST_WORKERS:4} \
           -m common \
           --prebuilt-client={toxinidir}/build_tmp/tensilelite/client/tensilelite-client \
           --global-parameters CheckASMCodeSize=True \
           {toxinidir}/Tensile/Tests/common {posargs}

    coverage report --show-missing

    python -c "print('\nHTML Report: htmlcov/index.html')"

[flake8]
exclude = .git,build*,dist,.cache,*~
max-line-length = 132
# E/W class violations are errors/warnings reported by pycodestyle, respectively
# This mostly just leaves pyflake errors.
ignore = E, W