TheRock/rocm-libraries/test/therock/test_miopen.py

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

import logging
import os
import shlex
import subprocess
from pathlib import Path
import platform

THEROCK_BIN_DIR = os.getenv("THEROCK_BIN_DIR")
SCRIPT_DIR = Path(__file__).resolve().parent
THEROCK_DIR = Path(
    os.environ.get("THEROCK_DIR") or SCRIPT_DIR.parent.parent.parent
).resolve()

AMDGPU_FAMILIES = os.getenv("AMDGPU_FAMILIES")
os_type = platform.system().lower()

# GTest sharding
SHARD_INDEX = os.getenv("SHARD_INDEX", 1)
TOTAL_SHARDS = os.getenv("TOTAL_SHARDS", 1)
environ_vars = os.environ.copy()
# For display purposes in the GitHub Action UI, the shard array is 1th indexed. However for shard indexes, we convert it to 0th index.
environ_vars["GTEST_SHARD_INDEX"] = str(int(SHARD_INDEX) - 1)
environ_vars["GTEST_TOTAL_SHARDS"] = str(TOTAL_SHARDS)

# Some of our runtime kernel compilations have been relying on either ROCM_PATH being set, or ROCm being installed at
# /opt/rocm. Neither of these is true in TheRock so we need to supply ROCM_PATH to our tests.
ROCM_PATH = Path(THEROCK_BIN_DIR).resolve().parent
environ_vars["ROCM_PATH"] = str(ROCM_PATH)

logging.basicConfig(level=logging.INFO)

###########################################

positive_filter = []
negative_filter = []

# Fusion #
positive_filter.append("*Fusion*")

# Batch Normalization #
positive_filter.append("*/GPU_BNBWD*_*")
positive_filter.append("*/GPU_BNOCLBWD*_*")
positive_filter.append("*/GPU_BNFWD*_*")
positive_filter.append("*/GPU_BNOCLFWD*_*")
positive_filter.append("*/GPU_BNInfer*_*")
positive_filter.append("*/GPU_BNActivInfer_*")
positive_filter.append("*/GPU_BNOCLInfer*_*")
positive_filter.append("*/GPU_bn_infer*_*")

# CPU tests
positive_filter.append("CPU_*")  # tests without a suite
positive_filter.append("*/CPU_*")  # tests with a suite

# Different
positive_filter.append("*/GPU_Cat_*")
positive_filter.append("*/GPU_ConvBiasActiv*")

# Convolutions
positive_filter.append("*/GPU_Conv*")
positive_filter.append("*/GPU_conv*")

# Solvers
positive_filter.append("*/GPU_UnitTestConv*")

# Misc

positive_filter.append("*/GPU_GetitemBwd*")
positive_filter.append("*/GPU_GLU_*")

positive_filter.append("*/GPU_GroupConv*")
positive_filter.append("*/GPU_GroupNorm_*")
positive_filter.append("*/GPU_GRUExtra_*")
positive_filter.append("*/GPU_TestActivation*")
positive_filter.append("*/GPU_HipBLASLtGEMMTest*")
positive_filter.append("*/GPU_KernelTuningNetTestConv*")
positive_filter.append("*/GPU_Kthvalue_*")
positive_filter.append("*/GPU_LayerNormTest*")
positive_filter.append("*/GPU_LayoutTransposeTest_*")
positive_filter.append("*/GPU_Lrn*")
positive_filter.append("*/GPU_lstm_extra*")

positive_filter.append("*/GPU_MultiMarginLoss_*")
positive_filter.append("*/GPU_ConvNonpack*")
positive_filter.append("*/GPU_PerfConfig_HipImplicitGemm*")
positive_filter.append("*/GPU_AsymPooling2d_*")
positive_filter.append("*/GPU_WidePooling2d_*")
positive_filter.append("*/GPU_PReLU_*")
positive_filter.append("*/GPU_Reduce*")
positive_filter.append("*/GPU_reduce_custom_*")
positive_filter.append("*/GPU_regression_issue_*")
positive_filter.append("*/GPU_RNNExtra_*")
positive_filter.append("*/GPU_RoPE*")
positive_filter.append("*/GPU_SoftMarginLoss*")
positive_filter.append("*/GPU_T5LayerNormTest_*")
positive_filter.append("*/GPU_Op4dTensorGenericTest_*")
positive_filter.append("*/GPU_TernaryTensorOps_*")
positive_filter.append("*/GPU_unaryTensorOps_*")
positive_filter.append("*/GPU_Transformers*")
positive_filter.append("*/GPU_TunaNetTest_*")
positive_filter.append("*/GPU_UnitTestActivationDescriptor_*")
positive_filter.append("*/GPU_FinInterfaceTest*")
positive_filter.append("*/GPU_VecAddTest_*")

positive_filter.append("*/GPU_KernelTuningNetTest*")

positive_filter.append("*/GPU_Bwd_Mha_*")
positive_filter.append("*/GPU_Fwd_Mha_*")
positive_filter.append("*/GPU_Softmax*")
positive_filter.append("*/GPU_Dropout*")
positive_filter.append("*/GPU_MhaBackward_*")
positive_filter.append("*/GPU_MhaForward_*")
positive_filter.append("*GPU_TestMhaFind20*")
positive_filter.append("*/GPU_MIOpenDriver*")

#############################################

negative_filter.append("*DeepBench*")
negative_filter.append("*MIOpenTestConv*")

# For sake of time saving on pre-commit step
####################################################
negative_filter.append("Full/GPU_MIOpenDriverConv2dTransTest*")  # 4 min 45 sec
negative_filter.append("Full/GPU_Reduce_FP64*")  # 4 min 19 sec
negative_filter.append("Full/GPU_BNOCLFWDTrainSerialRun3D_BFP16*")  # 3 min 37 sec
negative_filter.append("Full/GPU_Lrn_FP32*")  # 2 min 50 sec
negative_filter.append("Full/GPU_Lrn_FP16*")  # 2 min 20 sec
negative_filter.append("Full/GPU_BNOCLInferSerialRun3D_BFP16*")  # 2 min 19 sec
negative_filter.append("Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*")  # 1 min 55 sec
negative_filter.append("Smoke/GPU_BNOCLInferLarge2D_BFP16*")  # 1 min 48 sec
negative_filter.append("Full/GPU_BNOCLBWDSerialRun3D_BFP16*")  # 1 min 28 sec
negative_filter.append("Smoke/GPU_BNOCLBWDLarge2D_BFP16*")  # 1 min 19 sec

negative_filter.append("Full/GPU_UnitTestActivationDescriptor_FP32*")  # 1 min 23 sec
negative_filter.append("Full/GPU_UnitTestActivationDescriptor_FP16*")  # 1 min 0 sec

negative_filter.append(
    "Full/GPU_MIOpenDriverRegressionBigTensorTest_FP32*"
)  # 0 min 59 sec

negative_filter.append(
    "Smoke/GPU_BNOCLBWDLargeFusedActivation2D_BFP16*"
)  # 0 min 52 sec
negative_filter.append("Smoke/GPU_BNOCLBWDLargeFusedActivation2D_FP16*")  # 0 min 49 sec

negative_filter.append("Full/GPU_ConvGrpBiasActivInfer_BFP16*")  # 0 min 40 sec
negative_filter.append("Full/GPU_ConvGrpBiasActivInfer_FP32*")  # 0 min 38 sec
negative_filter.append("Full/GPU_ConvGrpBiasActivInfer_FP16*")  # 0 min 25 sec

negative_filter.append("Full/GPU_ConvGrpActivInfer_BFP16*")  # 0 min 42 sec
negative_filter.append("Full/GPU_ConvGrpActivInfer_FP32*")  # 0 min 35 sec
negative_filter.append("Full/GPU_ConvGrpActivInfer_FP16*")  # 0 min 25 sec

negative_filter.append("Full/GPU_ConvGrpBiasActivInfer3D_BFP16*")  # 0 min 27 sec
negative_filter.append("Full/GPU_ConvGrpBiasActivInfer3D_FP32*")  # 0 min 25 sec
negative_filter.append("Full/GPU_ConvGrpBiasActivInfer3D_FP16*")  # 0 min 19 sec

negative_filter.append("Full/GPU_ConvGrpActivInfer3D_BFP16*")  # 0 min 27 sec
negative_filter.append("Full/GPU_ConvGrpActivInfer3D_FP32*")  # 0 min 22 sec
negative_filter.append("Full/GPU_ConvGrpActivInfer3D_FP16*")  # 0 min 16 sec

# TODO(#3202): Re-enable tests once issues are resolved
TEST_TO_IGNORE = {
    "gfx110X-all": {
        "windows": [
            # Failed on gfx1103
            "Smoke/CPU_Handle_NONE.TestHIP/with_stream_false_test_id_0",
            "Full/GPU_reduce_custom_fp32_fp16_FP32.FloatTest_reduce_custom_fp32_fp16/1",
            "Full/GPU_reduce_custom_fp32_fp16_FP32.FloatTest_reduce_custom_fp32_fp16/5",
            "Full/GPU_reduce_custom_fp32_fp16_FP32.FloatTest_reduce_custom_fp32_fp16/9",
            "Full/GPU_reduce_custom_fp32_fp16_FP32.FloatTest_reduce_custom_fp32_fp16/13",
            "Full/GPU_reduce_custom_fp32_fp16_FP32.FloatTest_reduce_custom_fp32_fp16/17",
            "Full/GPU_reduce_custom_fp32_fp16_FP16.HalfTest_reduce_custom_fp32_fp16/1",
            "Full/GPU_reduce_custom_fp32_fp16_FP16.HalfTest_reduce_custom_fp32_fp16/5",
            "Full/GPU_reduce_custom_fp32_fp16_FP16.HalfTest_reduce_custom_fp32_fp16/9",
            "Full/GPU_reduce_custom_fp32_fp16_FP16.HalfTest_reduce_custom_fp32_fp16/13",
            "Full/GPU_reduce_custom_fp32_fp16_FP16.HalfTest_reduce_custom_fp32_fp16/17",
        ]
    },
    "gfx1151": {
        "windows": ["Full/GPU_UnitTestConvSolverGemmBwdRestBwd_FP16.GemmBwdRest/0"]
    },
    "gfx950-dcgpu": {"linux": ["*DBSync*"]},
}

if AMDGPU_FAMILIES in TEST_TO_IGNORE and os_type in TEST_TO_IGNORE[AMDGPU_FAMILIES]:
    ignored_tests = TEST_TO_IGNORE[AMDGPU_FAMILIES][os_type]
    negative_filter.extend(ignored_tests)

# Failing on on win gfx110x
if any(prefix in AMDGPU_FAMILIES for prefix in ["gfx110"]):
    negative_filter.append("*/GPU_MIOpenDriver*")
    negative_filter.append("Smoke/CPU_Handle_NONE*")
    negative_filter.append("Full/GPU_reduce_custom_fp32*")

# Tests to be filtered for navi
# 1- Ignore gfx942 tests
# TODO: There is no FP32 wmma on Navi, remove all FP32 conv tests. These should already be skipped via applicability for
# CK solvers

if any(prefix in AMDGPU_FAMILIES for prefix in ["gfx110", "gfx115", "gfx120"]):
    # These are ignored in miopen
    negative_filter.append(
        "Smoke/GPU_BNFWDTrainLargeFusedActivation2D_FP32.BnV2LargeFWD_TrainCKfp32Activation/NCHW_BNSpatial_testBNAPIV1_Dim_2_test_id_32"
    )  # Temporarily disabled until gfx1151 CI nodes have fw 31 or higher installed
    negative_filter.append(
        "Smoke/GPU_BNFWDTrainLarge2D_FP32.BnV2LargeFWD_TrainCKfp32/NCHW_BNSpatial_testBNAPIV2_Dim_2_test_id_64"
    )  # Temporarily disabled until gfx1151 CI nodes have fw 31 or higher installed
    # this could address 2
    negative_filter.append(
        "*SerialRun3D*"
    )  # These FP32 SerialRun3D tests use so much memory that they have a risk of timing out the machine during tests
    # this could address 1
    negative_filter.append("*gfx942*")
    # List of currently failing tests
    negative_filter.append("*GPU_UnitTestConvSolverFFTFwd_FP32*")
    negative_filter.append("*GPU_UnitTestConvSolverFFTBwd_FP32*")
    negative_filter.append("*GPU_TernaryTensorOps_FP64*")
    negative_filter.append("*GPU_TernaryTensorOps_FP16*")
    negative_filter.append("*GPU_TernaryTensorOps_FP32*")
    negative_filter.append("*GPU_Op4dTensorGenericTest_FP32*")
    negative_filter.append("*GPU_UnitTestActivationDescriptor_FP16*")
    negative_filter.append("*GPU_UnitTestActivationDescriptor_FP32*")
    negative_filter.append("*CPU_TuningPolicy_NONE*")
    negative_filter.append("*GPU_Dropout_FP32*")
    negative_filter.append("*GPU_Dropout_FP16*")

    # TODO: We need to work to re-enable these
    negative_filter.append(
        "*GPU_GroupConv3D_BackwardData_FP16.GroupConv3D_BackwardData_half_Test*"
    )
    negative_filter.append(
        "*GPU_GroupConv3D_BackwardData_BFP16.GroupConv3D_BackwardData_bfloat16_Test*"
    )
    negative_filter.append(
        "*GPU_UnitTestConvSolverImplicitGemmGroupWrwXdlops_BFP16.ConvHipImplicitGemmGroupWrwXdlops*"
    )

    negative_filter.append("Smoke/GPU_MultiMarginLoss*")

    negative_filter.append(
        "*CPU_UnitTestConvSolverImplicitGemmGroupWrwXdlopsDevApplicability_FP16.ConvHipImplicitGemmGroupWrwXdlops*"
    )

    # Disable long running tests
    negative_filter.append("Full/GPU_Softmax_FP32*")  # 24 min
    negative_filter.append("Full/GPU_Softmax_BFP16*")  # 13 min
    negative_filter.append("Full/GPU_Softmax_FP16*")  # 11.5 min
    negative_filter.append("Smoke/GPU_Reduce_FP32*")  # 6.5 min
    negative_filter.append("Smoke/GPU_Reduce_FP16*")  # 4.5 min

####################################################

# Creating a quick test filter
quick_filter = [
    # Batch norm FWD smoke tests
    "Smoke/GPU_BNCKFWDTrainLarge2D_FP16*",
    "Smoke/GPU_BNOCLFWDTrainLarge2D_FP16*",
    "Smoke/GPU_BNOCLFWDTrainLarge3D_FP16*",
    "Smoke/GPU_BNCKFWDTrainLarge2D_BFP16*",
    "Smoke/GPU_BNOCLFWDTrainLarge2D_BFP16*",
    "Smoke/GPU_BNOCLFWDTrainLarge3D_BFP16*",
    # CK Grouped FWD Conv smoke tests
    "Smoke/GPU_UnitTestConvSolverImplicitGemmFwdXdlops_FP16*",
    "Smoke/GPU_UnitTestConvSolverImplicitGemmFwdXdlops_BFP16*",
]

# TODO(rocm-libraries#2266): re-enable test for gfx950-dcgpu
if AMDGPU_FAMILIES != "gfx950-dcgpu":
    quick_filter.append("*DBSync*")
    positive_filter.append("*DBSync*")

####################################################

# If quick tests are enabled, we run quick tests only.
# Otherwise, we run the normal test suite
test_type = os.getenv("TEST_TYPE", "full")
if test_type == "quick":
    test_filter = "--gtest_filter=" + ":".join(quick_filter)
else:
    test_filter = (
        "--gtest_filter=" + ":".join(positive_filter) + "-" + ":".join(negative_filter)
    )
#############################################

cmd = [f"{THEROCK_BIN_DIR}/miopen_gtest", test_filter]
logging.info(f"++ Exec [{THEROCK_DIR}]$ {shlex.join(cmd)}")
subprocess.run(cmd, cwd=THEROCK_DIR, check=True, env=environ_vars)