CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
344 lines
8.3 KiB
C++
344 lines
8.3 KiB
C++
// Copyright Advanced Micro Devices, Inc., or its affiliates.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
#ifdef ROCROLLER_USE_HIP
|
|
#include <hip/hip_ext.h>
|
|
#include <hip/hip_runtime.h>
|
|
#endif /* ROCROLLER_USE_HIP */
|
|
|
|
#include "GEMMTestBase.hpp"
|
|
|
|
namespace GEMMTests
|
|
{
|
|
using namespace rocRoller;
|
|
namespace SolutionParams = rocRoller::Parameters::Solution;
|
|
|
|
// ========================================================================
|
|
// GEMMJammedTestSuite
|
|
// ========================================================================
|
|
|
|
class GEMMJammedTestSuite : public BaseGEMMContextFixture<>
|
|
{
|
|
};
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_2x2)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 2 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 4;
|
|
|
|
gemm.loadPathA = SolutionParams::LoadPath::BufferToVGPR;
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
gemm.fuseLoops = false;
|
|
|
|
basicGEMM<Half>(gemm);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_2x1)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 128;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 2 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 4;
|
|
|
|
gemm.betaInFma = false;
|
|
|
|
gemm.transA = "T";
|
|
gemm.transB = "N";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b64"), 18);
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_read_b128"), 8);
|
|
EXPECT_EQ(countSubstring(generatedCode, "buffer_store_dwordx4"), 8);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_2x1_UnrollK)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 128;
|
|
gemm.macK = 16;
|
|
|
|
gemm.unrollK = 2;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 2 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 4;
|
|
|
|
gemm.transA = "T";
|
|
gemm.transB = "N";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b64"), 22);
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_read_b128"), 8);
|
|
EXPECT_EQ(countSubstring(generatedCode, "buffer_store_dwordx4"), 8);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_1x2)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 128;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 4 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 2;
|
|
|
|
gemm.transA = "T";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b64"), 18);
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_read_b128"), 8);
|
|
EXPECT_EQ(countSubstring(generatedCode, "buffer_store_dwordx4"), 8);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_1x2_UnrollK)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 128;
|
|
gemm.macK = 16;
|
|
|
|
gemm.unrollK = 4;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 4 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 2;
|
|
|
|
gemm.transA = "T";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b64"), 26);
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_read_b128"), 8);
|
|
EXPECT_EQ(countSubstring(generatedCode, "buffer_store_dwordx4"), 8);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_1x8)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 4 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 1;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
basicGEMM<Half>(gemm);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_1x8_UnrollK)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.unrollK = 2;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 4 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 1;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
basicGEMM<Half>(gemm);
|
|
}
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_2x4)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 2 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 2;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b128"), 3);
|
|
EXPECT_EQ(countSubstring(generatedCode, "v_pack_b32_f16"), 152);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_2x4_UnrollK)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.unrollK = 2;
|
|
|
|
gemm.prefetchInFlight = 2;
|
|
gemm.prefetchLDSFactor = 2;
|
|
gemm.prefetchMixMemOps = true;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 2 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 2;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b128"), 9);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_4x2)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 1 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 4;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
gemm.transB = "N";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b128"), 3);
|
|
}
|
|
|
|
TEST_P(GEMMJammedTestSuite, GPU_GEMM_Jammed_4x2_UnrollK)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasMFMA);
|
|
GEMMProblem gemm;
|
|
|
|
gemm.m = 256;
|
|
gemm.n = 512;
|
|
gemm.k = 64;
|
|
|
|
gemm.macM = 128;
|
|
gemm.macN = 256;
|
|
gemm.macK = 16;
|
|
|
|
gemm.unrollK = 4;
|
|
|
|
gemm.waveK = 8;
|
|
|
|
gemm.workgroupSizeX = 1 * gemm.wavefrontSize;
|
|
gemm.workgroupSizeY = 4;
|
|
|
|
gemm.storePath = SolutionParams::StorePath::VGPRToGlobalMemoryWithBuffer;
|
|
|
|
gemm.transB = "N";
|
|
|
|
basicGEMM<Half>(gemm);
|
|
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
|
|
EXPECT_EQ(countSubstring(generatedCode, "ds_write_b128"), 15);
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(GEMMJammedTest, GEMMJammedTestSuite, currentGPUISA());
|
|
|
|
} // namespace GEMMTests
|