TheRock/math-libs/libhipcxx/.upstream-tests/utils/libcudacxx/test/format.py

# ===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===----------------------------------------------------------------------===##

# MIT License
#
# Modifications Copyright (C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import copy
import errno
import os
import time

import lit.Test  # pylint: disable=import-error
import lit.TestRunner  # pylint: disable=import-error
import lit.util
from lit.TestRunner import IntegratedTestKeywordParser, ParserKind

import libcudacxx.util

# pylint: disable=import-error
from libcudacxx.test.executor import LocalExecutor as LocalExecutor


class LibcxxTestFormat(object):
    """
    Custom test format handler for use with the test format use by libc++.

    Tests fall into two categories:
      FOO.pass.cpp - Executable test which should compile, run, and exit with
                     code 0.
      FOO.fail.cpp - Negative test case which is expected to fail compilation.
      FOO.runfail.cpp - Negative test case which is expected to compile, run,
                        and exit with non-zero exit code.
      FOO.sh.cpp   - A test that uses LIT's ShTest format.
    """

    def __init__(self, cxx, use_verify_for_fail, execute_external, executor, exec_env):
        self.cxx = copy.deepcopy(cxx)
        self.use_verify_for_fail = use_verify_for_fail
        self.execute_external = execute_external
        self.executor = executor
        self.exec_env = dict(exec_env)

    @staticmethod
    def _make_custom_parsers():
        return [
            IntegratedTestKeywordParser(
                "FLAKY_TEST.", ParserKind.TAG, initial_value=False
            ),
            IntegratedTestKeywordParser(
                "MODULES_DEFINES:", ParserKind.LIST, initial_value=[]
            ),
        ]

    @staticmethod
    def _get_parser(key, parsers):
        for p in parsers:
            if p.keyword == key:
                return p
        assert False and "parser not found"

    # TODO: Move this into lit's FileBasedTest
    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
        source_path = testSuite.getSourcePath(path_in_suite)
        for filename in os.listdir(source_path):
            # Ignore dot files and excluded tests.
            if filename.startswith(".") or filename in localConfig.excludes:
                continue

            filepath = os.path.join(source_path, filename)
            if not os.path.isdir(filepath):
                if any([filename.endswith(ext) for ext in localConfig.suffixes]):
                    yield lit.Test.Test(
                        testSuite, path_in_suite + (filename,), localConfig
                    )

    def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
        yield lit.Test.Test(testSuite, path_in_suite, localConfig)

    def execute(self, test, lit_config):
        while True:
            try:
                return self._execute(test, lit_config)
            except OSError as oe:
                if oe.errno != errno.ETXTBSY:
                    raise
                time.sleep(0.1)

    def _execute(self, test, lit_config):
        name = test.path_in_suite[-1]
        name_root, name_ext = os.path.splitext(name)
        is_libcxx_test = test.path_in_suite[0] == "libcxx"
        is_sh_test = name_root.endswith(".sh")
        is_pass_test = name.endswith(".pass.cpp") or name.endswith(".pass.mm")
        is_fail_test = name.endswith(".fail.cpp") or name.endswith(".fail.mm")
        is_runfail_test = name.endswith(".runfail.cpp") or name.endswith(".runfail.mm")
        assert (
            is_sh_test or name_ext == ".cpp" or name_ext == ".mm"
        ), "non-cpp file must be sh test"

        if test.config.unsupported:
            return (lit.Test.UNSUPPORTED, "A lit.local.cfg marked this unsupported")

        parsers = self._make_custom_parsers()
        script = lit.TestRunner.parseIntegratedTestScript(
            test, additional_parsers=parsers, require_script=is_sh_test
        )
        # Check if a result for the test was returned. If so return that
        # result.
        if isinstance(script, lit.Test.Result):
            return script
        if lit_config.noExecute:
            # if we expect the test to fail at runtime, XFAIL is the proper return value if we never run the test
            if test.xfails:
                return lit.Test.Result(lit.Test.XFAIL)
            return lit.Test.Result(lit.Test.PASS)

        # Check that we don't have run lines on tests that don't support them.
        if not is_sh_test and len(script) != 0:
            lit_config.fatal("Unsupported RUN line found in test %s" % name)

        tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
        substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
        script = lit.TestRunner.applySubstitutions(script, substitutions)

        test_cxx = copy.deepcopy(self.cxx)
        if is_fail_test:
            test_cxx.useCCache(False)
            test_cxx.useWarnings(False)
        extra_modules_defines = self._get_parser("MODULES_DEFINES:", parsers).getValue()
        if "-fmodules" in test.config.available_features:
            test_cxx.compile_flags += [
                ("-D%s" % mdef.strip()) for mdef in extra_modules_defines
            ]
            test_cxx.addWarningFlagIfSupported("-Wno-macro-redefined")
            # FIXME: libc++ debug tests #define _CCCL_ASSERT to override it
            # If we see this we need to build the test against uniquely built
            # modules.
            if is_libcxx_test:
                with open(test.getSourcePath(), "rb") as f:
                    contents = f.read()
                if b"#define _CCCL_ASSERT" in contents:
                    test_cxx.useModules(False)

        # Dispatch the test based on its suffix.
        if is_sh_test:
            if not isinstance(self.executor, LocalExecutor):
                # We can't run ShTest tests with a executor yet.
                # For now, bail on trying to run them
                return lit.Test.UNSUPPORTED, "ShTest format not yet supported"
            test.config.environment = dict(self.exec_env)
            return lit.TestRunner._runShTest(
                test, lit_config, self.execute_external, script, tmpBase
            )
        elif is_fail_test:
            return self._evaluate_fail_test(test, test_cxx, parsers)
        elif is_pass_test:
            return self._evaluate_pass_test(
                test, tmpBase, lit_config, test_cxx, parsers
            )
        elif is_runfail_test:
            return self._evaluate_pass_test(
                test, tmpBase, lit_config, test_cxx, parsers, run_should_pass=False
            )
        else:
            # No other test type is supported
            assert False

    def _clean(self, exec_path):  # pylint: disable=no-self-use
        libcudacxx.util.cleanFile(exec_path)

    def _evaluate_pass_test(
        self, test, tmpBase, lit_config, test_cxx, parsers, run_should_pass=True
    ):
        execDir = os.path.dirname(test.getExecPath())
        source_path = test.getSourcePath()
        exec_path = tmpBase + ".exe"
        object_path = tmpBase + ".o"
        # Create the output directory if it does not already exist.
        libcudacxx.util.mkdir_p(os.path.dirname(tmpBase))
        try:
            is_flaky = self._get_parser("FLAKY_TEST.", parsers).getValue()
            # TODO(AMD): set this to 1 again when tests have stabilized with TheRock
            # (https://github.com/ROCm/libhipcxx/issues/14)
            max_retry = 3 #if is_flaky else 1
            time_out_detected = False
            compile_cmd = "None"
            out = "None"
            retry_count = -1
            while retry_count < max_retry:
                retry_count += 1
                try:
                    # Compile the test
                    cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
                        source_path, out=exec_path, object_file=object_path,
                        cwd=execDir)
                    compile_cmd = cmd
                    if rc != 0:
                        report = libcudacxx.util.makeReport(cmd, out, err, rc)
                        report += "Compilation failed unexpectedly!"
                        if retry_count >= max_retry:
                            return lit.Test.Result(lit.Test.FAIL, report)
                        else:
                            continue
                    # Run the test
                    local_cwd = os.path.dirname(source_path)
                    env = None
                    if self.exec_env:
                        env = self.exec_env
                    # TODO: Only list actually needed files in file_deps.
                    # Right now we just mark all of the .dat files in the same
                    # directory as dependencies, but it's likely less than that. We
                    # should add a `// FILE-DEP: foo.dat` to each test to track this.
                    data_files = [
                        os.path.join(local_cwd, f)
                        for f in os.listdir(local_cwd)
                        if f.endswith(".dat")
                    ]
                    cmd, out, err, rc = self.executor.run(
                        exec_path, [exec_path], local_cwd, data_files, env
                    )
                    report = "Compiled With: '%s'\n" % " ".join(compile_cmd)
                    report += libcudacxx.util.makeReport(cmd, out, err, rc)
                    result_expected = (rc == 0) == run_should_pass
                except (libcudacxx.util.ExecuteCommandTimeoutException, lit.util.ExecuteCommandTimeoutException) as e:
                    err = str(e)
                    rc = -42
                    out = "None"
                    report = "Compiled With: '%s'\n" % " ".join(compile_cmd)
                    report += libcudacxx.util.makeReport(compile_cmd, out, err, rc)
                    result_expected = False
                    time_out_detected = True
                if result_expected:
                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
                    return lit.Test.Result(res, report)
                # Rarely devices are unavailable, so just restart the test to avoid false negatives.
                elif (
                    rc != 0 and "hipErrorNoDevice" in out and max_retry <= 5
                ):
                    max_retry += 1
                elif retry_count >= max_retry:
                    if run_should_pass:
                        report += f"Compiled test failed unexpectedly after {retry_count} retry attempt(s)!"
                    else:
                        report += f"Compiled test succeeded unexpectedly after {retry_count} retry attempt(s)!"
                    if time_out_detected:
                        return lit.Test.Result(lit.Test.TIMEOUT, report)
                    return lit.Test.Result(lit.Test.FAIL, report)

            assert False  # Unreachable
        finally:
            # Note that cleanup of exec_file happens in `_clean()`. If you
            # override this, cleanup is your responsibility.
            libcudacxx.util.cleanFile(object_path)
            self._clean(exec_path)

    def _evaluate_fail_test(self, test, test_cxx, parsers):
        source_path = test.getSourcePath()
        # FIXME: lift this detection into LLVM/LIT.
        with open(source_path, "rb") as f:
            contents = f.read()
        verify_tags = [
            b"expected-note",
            b"expected-remark",
            b"expected-warning",
            b"expected-error",
            b"expected-no-diagnostics",
        ]
        use_verify = self.use_verify_for_fail and any(
            [tag in contents for tag in verify_tags]
        )
        # FIXME(EricWF): GCC 5 does not evaluate static assertions that
        # are dependant on a template parameter when '-fsyntax-only' is passed.
        # This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
        # when using Clang.
        if test_cxx.type != "gcc" and test_cxx.type != "nvcc":
            test_cxx.flags += ["-fsyntax-only"]
        if use_verify:
            test_cxx.useVerify()
            test_cxx.useWarnings()
            if "-Wuser-defined-warnings" in test_cxx.warning_flags:
                test_cxx.warning_flags += ["-Wno-error=user-defined-warnings"]
        else:
            # We still need to enable certain warnings on .fail.cpp test when
            # -verify isn't enabled. Such as -Werror=unused-result. However,
            # we don't want it enabled too liberally, which might incorrectly
            # allow unrelated failure tests to 'pass'.
            #
            # Therefore, we check if the test was expected to fail because of
            # nodiscard before enabling it
            test_str_list = [b"ignoring return value", b"nodiscard", b"NODISCARD"]
            if any(test_str in contents for test_str in test_str_list):
                if test_cxx.type != "nvc++":
                    test_cxx.flags += [
                        "-Xcompiler",
                        "-Werror",
                        "-Xcompiler",
                        "-Wunused",
                    ]
                else:
                    test_cxx.flags += ["-Xcompiler", "-Werror=unused-result"]
        cmd = ""
        out = "None"
        report = ""
        # TODO(AMD): set this to 1 again when tests have stabilized with TheRock
        # (https://github.com/ROCm/libhipcxx/issues/14)
        max_retry = 3 #if is_flaky else 1
        time_out_detected = False
        for retry_count in range(max_retry):
            try:
                cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
            except (libcudacxx.util.ExecuteCommandTimeoutException, lit.util.ExecuteCommandTimeoutException) as e:
                err = str(e)
                rc = -42
                out = "None"
                result_expected = False
                time_out_detected = True

            def check_rc(rc):
                return rc == 0 if use_verify else rc != 0
                
            report = libcudacxx.util.makeReport(cmd, out, err, rc)
            if time_out_detected:
                if retry_count + 1 == max_retry:
                    return lit.Test.Result(lit.Test.TIMEOUT, report)
                else:
                    continue
            else:
                if check_rc(rc):
                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
                    return lit.Test.Result(res, report)
                else:
                    report += (
                        "Expected compilation to fail!\n"
                        if not use_verify
                        else "Expected compilation using verify to pass!\n"
                    )
                    return lit.Test.Result(lit.Test.FAIL, report)