derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
763 lines
34 KiB
C++
763 lines
34 KiB
C++
/*******************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright 2025 AMD ROCm(TM) Software
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*
|
|
*******************************************************************************/
|
|
|
|
#include <rocRoller/KernelOptions_detail.hpp>
|
|
|
|
#include "MatrixMultiplyTestBase.hpp"
|
|
|
|
#include "Utilities.hpp"
|
|
|
|
using namespace rocRoller;
|
|
|
|
namespace MatrixMultiplyTest
|
|
{
|
|
namespace SolutionParams = rocRoller::Parameters::Solution;
|
|
// Params are: (AB type, waveK), (transA, transB)
|
|
class WMMATestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<
|
|
std::tuple<std::pair<rocRoller::DataType, int>, std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params are: (AB type, waveK), (transA, transB)
|
|
class F16AccWMMATestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<
|
|
std::tuple<std::pair<rocRoller::DataType, int>, std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params are: A type, B type, waveK, (transA, transB)
|
|
class MixedWMMATestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<std::tuple<rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
int,
|
|
std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params are: A type, B type, waveK, (transA, transB)
|
|
class MixedWMMAF8F6F4TestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<std::tuple<rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
int,
|
|
std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params are: A type, B type, A scale type, B scale type, waveK, scaleBlockSize, (transA, transB)
|
|
class MixedWMMAF8F6F4ScaledTestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<std::tuple<rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
int,
|
|
uint,
|
|
std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params are: (transA, transB)
|
|
class WMMAF4TestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<std::pair<std::string, std::string>>
|
|
{
|
|
};
|
|
|
|
// Params are: A typeA, B type, A scale type, B scale type, scaleBlockSize, (transA, transB)
|
|
class WMMAF4ScaledTestGFX1250
|
|
: public BaseMatrixMultiplyContextFixture<std::tuple<rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
rocRoller::DataType,
|
|
uint,
|
|
std::pair<std::string, std::string>>>
|
|
{
|
|
};
|
|
|
|
// Params: waveK
|
|
class ABCWMMATestGFX1250 : public BaseMatrixMultiplyContextFixture<int>
|
|
{
|
|
};
|
|
|
|
TEST_P(WMMATestGFX1250, GPU_MatrixMultiplyMacroTileWMMA)
|
|
{
|
|
const auto [typeAndWaveK, transOp] = std::get<1>(GetParam());
|
|
const auto [typeAB, waveK] = typeAndWaveK;
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
auto typeStr{"f16"};
|
|
switch(typeAB)
|
|
{
|
|
case DataType::Half:
|
|
matrixMultiplyMacroTile<Half, Half, float>(16, 16, waveK, 1, loadPathB, transA, transB);
|
|
break;
|
|
case DataType::BFloat16:
|
|
matrixMultiplyMacroTile<BFloat16, BFloat16, float>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf16";
|
|
break;
|
|
case DataType::Float:
|
|
matrixMultiplyMacroTile<float, float, float>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "f32";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format(
|
|
"Unexpected data type: {}. (Allowed: Half, Bfloat16, or Float)", toString(typeAB)));
|
|
};
|
|
|
|
const auto numWMMAs = 4; // F32 & F16: mac_k = 4 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f32_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(F16AccWMMATestGFX1250, GPU_MatrixMultiplyMacroTileWMMA)
|
|
{
|
|
const auto [typeAndWaveK, transOp] = std::get<1>(GetParam());
|
|
const auto [dataType, waveK] = typeAndWaveK;
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
auto typeStr{"f16"};
|
|
switch(dataType)
|
|
{
|
|
case DataType::Half:
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_f16_16x16x32_f16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyMacroTile<Half, Half, Half, Half>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
break;
|
|
case DataType::BFloat16:
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_bf16_16x16x32_bf16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyMacroTile<BFloat16, BFloat16, BFloat16, BFloat16>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf16";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: Half and Bfloat16)",
|
|
toString(dataType)));
|
|
};
|
|
|
|
const auto numWMMAs = 4; // F16 mac_k = 4 * wave_k
|
|
const std::string wmmaMnemonic{
|
|
fmt::format("v_wmma_{}_16x16x{}_{}", typeStr, waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(WMMATestGFX1250, GPU_MatrixMultiplyABWMMA)
|
|
{
|
|
const auto [typeAndWaveK, transOp] = std::get<1>(GetParam());
|
|
const auto [typeAB, waveK] = typeAndWaveK;
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
auto typeStr{"f16"};
|
|
switch(typeAB)
|
|
{
|
|
case DataType::Half:
|
|
matrixMultiplyAB<Half, Half, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
break;
|
|
case DataType::BFloat16:
|
|
matrixMultiplyAB<BFloat16, BFloat16, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf16";
|
|
break;
|
|
case DataType::Float:
|
|
matrixMultiplyAB<float, float, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "f32";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format(
|
|
"Unexpected data type: {}. (Allowed: Half, Bfloat16, or Float)", toString(typeAB)));
|
|
};
|
|
|
|
const auto numWMMAs = 2; // mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f32_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(F16AccWMMATestGFX1250, GPU_MatrixMultiplyABWMMA)
|
|
{
|
|
const auto [typeAndWaveK, transOp] = std::get<1>(GetParam());
|
|
const auto [typeAB, waveK] = typeAndWaveK;
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
auto typeStr{"f16"};
|
|
switch(typeAB)
|
|
{
|
|
case DataType::Half:
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_f16_16x16x32_f16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyAB<Half, Half, Half, Half>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
break;
|
|
case DataType::BFloat16:
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_bf16_16x16x32_bf16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyAB<BFloat16, BFloat16, BFloat16, BFloat16>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf16";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: Half and Bfloat16)",
|
|
toString(typeAB)));
|
|
};
|
|
|
|
const auto numWMMAs = 2; // mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{
|
|
fmt::format("v_wmma_{}_16x16x{}_{}", typeStr, waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMATestGFX1250, GPU_MatrixMultiplyMacroTileMixedWMMA)
|
|
{
|
|
const auto [typeA, typeB, waveK, transOp] = std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
std::string typeStr;
|
|
|
|
KernelOptions options{};
|
|
options->favourF8F6F4OverF8MatrixInstruction = false;
|
|
setKernelOptions(options);
|
|
|
|
if(typeA == typeB)
|
|
{
|
|
switch(typeA)
|
|
{
|
|
case DataType::FP8:
|
|
matrixMultiplyMacroTile<FP8, FP8, float>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "fp8_fp8";
|
|
break;
|
|
case DataType::BF8:
|
|
matrixMultiplyMacroTile<BF8, BF8, float>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf8_bf8";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: FP8 and BF8)",
|
|
toString(typeA)));
|
|
};
|
|
}
|
|
else if(typeA == DataType::FP8)
|
|
{
|
|
AssertFatal(typeB == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: BF8)");
|
|
matrixMultiplyMacroTile<FP8, BF8, float>(16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "fp8_bf8";
|
|
}
|
|
else
|
|
{
|
|
AssertFatal(typeA == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeA) + "(Allowed: BF8)");
|
|
AssertFatal(typeB == DataType::FP8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: FP8)");
|
|
matrixMultiplyMacroTile<BF8, FP8, float>(16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf8_fp8";
|
|
}
|
|
|
|
const auto numWMMAs = 2; // F8 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f32_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMATestGFX1250, GPU_MatrixMultiplyMacroTileMixedF16AccWMMA)
|
|
{
|
|
REQUIRE_ANY_OF_ARCH_CAP(GPUCapability::HasWMMA_f16_16x16x64_f8,
|
|
GPUCapability::HasWMMA_f16_16x16x128_f8);
|
|
|
|
const auto [typeA, typeB, waveK, transOp] = std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
std::string typeStr;
|
|
|
|
KernelOptions options{};
|
|
options->favourF8F6F4OverF8MatrixInstruction = false;
|
|
setKernelOptions(options);
|
|
|
|
if(typeA == typeB)
|
|
{
|
|
switch(typeA)
|
|
{
|
|
case DataType::FP8:
|
|
matrixMultiplyMacroTile<FP8, FP8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "fp8_fp8";
|
|
break;
|
|
case DataType::BF8:
|
|
matrixMultiplyMacroTile<BF8, BF8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf8_bf8";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: FP8 and BF8)",
|
|
toString(typeA)));
|
|
};
|
|
}
|
|
else if(typeA == DataType::FP8)
|
|
{
|
|
AssertFatal(typeB == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: BF8)");
|
|
matrixMultiplyMacroTile<FP8, BF8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "fp8_bf8";
|
|
}
|
|
else
|
|
{
|
|
AssertFatal(typeA == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeA) + "(Allowed: BF8)");
|
|
AssertFatal(typeB == DataType::FP8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: FP8)");
|
|
matrixMultiplyMacroTile<BF8, FP8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathB, transA, transB);
|
|
typeStr = "bf8_fp8";
|
|
}
|
|
|
|
const auto numWMMAs = 2; // F8 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f16_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMAF8F6F4TestGFX1250, GPU_MatrixMultiplyMacroTileMixedWMMA)
|
|
{
|
|
const auto [typeA, typeB, waveK, transOp] = std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
|
|
matrixMultiplyMacroTileMixed(typeA, typeB, 16, 16, waveK, 1, loadPathB, transA, transB);
|
|
|
|
const auto numWMMAs = 2; // F8, F6, and F4 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f32_16x16x{}_f8f6f4", waveK)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(WMMAF4TestGFX1250, GPU_MatrixMultiplyMacroTileWMMA)
|
|
{
|
|
const auto [transA, transB] = std::get<1>(GetParam());
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
|
|
matrixMultiplyMacroTile<FP4, FP4, float>(32, 16, 128, 1, loadPathB, transA, transB);
|
|
|
|
const auto numWMMAs = 2; // F4 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{"v_wmma_f32_32x16x128_f4"};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(WMMAF4ScaledTestGFX1250, GPU_MatrixMultiplyMacroTileWMMA)
|
|
{
|
|
const auto [typeA, typeB, scaleTypeA, scaleTypeB, scaleBlockSize, transOp]
|
|
= std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const ScaleParams scaleParams{scaleTypeA, scaleTypeB, scaleBlockSize};
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
|
|
matrixMultiplyMacroTileMixed(
|
|
typeA, typeB, 32, 16, 128, 1, loadPathB, transA, transB, scaleParams);
|
|
|
|
const auto numWMMAs = 2; // F4 mac_k = 2 * wave_k
|
|
std::string wmmaMnemonic;
|
|
if(scaleBlockSize == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasBlockScaling32);
|
|
wmmaMnemonic = "v_wmma_scale_f32_32x16x128_f4";
|
|
}
|
|
else if(scaleBlockSize == 16)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasBlockScaling16);
|
|
wmmaMnemonic = "v_wmma_scale16_f32_32x16x128_f4";
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>(
|
|
fmt::format("Unsupported scaleBlockSize: {}. (Allowed 16, 32)", scaleBlockSize));
|
|
}
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMAF8F6F4ScaledTestGFX1250, GPU_ScaledMatrixMultiplyMacroTileF8F6F4)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_scale_f8f6f4);
|
|
|
|
const auto [typeA, typeB, scaleTypeA, scaleTypeB, waveK, scaleBlockSize, transOp]
|
|
= std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const ScaleParams scaleParams = {
|
|
.scaleTypeA = scaleTypeA, .scaleTypeB = scaleTypeB, .scaleBlockSize = scaleBlockSize};
|
|
const auto loadPathB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
|
|
matrixMultiplyMacroTileMixed(
|
|
typeA, typeB, 16, 16, waveK, 1, loadPathB, transA, transB, scaleParams);
|
|
|
|
const auto numWMMAs = 2; // F8, F6, and F4 mac_k = 2 * wave_k
|
|
std::string wmmaMnemonic;
|
|
if(scaleBlockSize == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasBlockScaling32);
|
|
wmmaMnemonic = fmt::format("v_wmma_scale_f32_16x16x{}_f8f6f4", waveK);
|
|
}
|
|
else if(scaleBlockSize == 16)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasBlockScaling16);
|
|
wmmaMnemonic = fmt::format("v_wmma_scale16_f32_16x16x{}_f8f6f4", waveK);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>(
|
|
fmt::format("Unsupported scaleBlockSize: {}. (Allowed 16, 32)", scaleBlockSize));
|
|
}
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMATestGFX1250, GPU_MatrixMultiplyABMixedWMMA)
|
|
{
|
|
const auto [typeA, typeB, waveK, transOp] = std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
std::string typeStr;
|
|
|
|
KernelOptions options{};
|
|
options->favourF8F6F4OverF8MatrixInstruction = false;
|
|
setKernelOptions(options);
|
|
|
|
if(typeA == typeB)
|
|
{
|
|
switch(typeA)
|
|
{
|
|
case DataType::FP8:
|
|
matrixMultiplyAB<FP8, FP8, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "fp8_fp8";
|
|
break;
|
|
case DataType::BF8:
|
|
matrixMultiplyAB<BF8, BF8, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf8_bf8";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: FP8 and BF8)",
|
|
toString(typeA)));
|
|
};
|
|
}
|
|
else if(typeA == DataType::FP8)
|
|
{
|
|
AssertFatal(typeB == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: BF8)");
|
|
matrixMultiplyAB<FP8, BF8, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "fp8_bf8";
|
|
}
|
|
else
|
|
{
|
|
AssertFatal(typeA == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeA) + "(Allowed: BF8)");
|
|
AssertFatal(typeB == DataType::FP8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: FP8)");
|
|
matrixMultiplyAB<BF8, FP8, float>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf8_fp8";
|
|
}
|
|
|
|
const auto numWMMAs = 2; // F8 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f32_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(MixedWMMATestGFX1250, GPU_MatrixMultiplyABMixedF16AccWMMA)
|
|
{
|
|
REQUIRE_ANY_OF_ARCH_CAP(GPUCapability::HasWMMA_f16_16x16x64_f8,
|
|
GPUCapability::HasWMMA_f16_16x16x128_f8);
|
|
|
|
const auto [typeA, typeB, waveK, transOp] = std::get<1>(GetParam());
|
|
const auto [transA, transB] = transOp;
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
std::string typeStr;
|
|
|
|
KernelOptions options{};
|
|
options->favourF8F6F4OverF8MatrixInstruction = false;
|
|
setKernelOptions(options);
|
|
|
|
if(typeA == typeB)
|
|
{
|
|
switch(typeA)
|
|
{
|
|
case DataType::FP8:
|
|
matrixMultiplyAB<FP8, FP8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "fp8_fp8";
|
|
break;
|
|
case DataType::BF8:
|
|
matrixMultiplyAB<BF8, BF8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf8_bf8";
|
|
break;
|
|
default:
|
|
Throw<FatalError>(fmt::format("Unexpected data type: {}. (Allowed: FP8 and BF8)",
|
|
toString(typeA)));
|
|
};
|
|
}
|
|
else if(typeA == DataType::FP8)
|
|
{
|
|
AssertFatal(typeB == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: BF8)");
|
|
matrixMultiplyAB<FP8, BF8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "fp8_bf8";
|
|
}
|
|
else
|
|
{
|
|
AssertFatal(typeA == DataType::BF8,
|
|
"Unexpected data type: " + ShowValue(typeA) + "(Allowed: BF8)");
|
|
AssertFatal(typeB == DataType::FP8,
|
|
"Unexpected data type: " + ShowValue(typeB) + "(Allowed: FP8)");
|
|
matrixMultiplyAB<BF8, FP8, Half, Half>(
|
|
16, 16, waveK, 1, loadPathAB, transA == "T", transB == "T");
|
|
typeStr = "bf8_fp8";
|
|
}
|
|
|
|
const auto numWMMAs = 2; // F8 mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f16_16x16x{}_{}", waveK, typeStr)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(ABCWMMATestGFX1250, GPU_MatrixMultiplyABCF16AccWMMAFP16)
|
|
{
|
|
const auto waveK = std::get<1>(GetParam());
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_F16_ACC);
|
|
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_f16_16x16x32_f16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyABC<Half, Half>(16, 16, waveK, 1, loadPathAB);
|
|
|
|
const auto numWMMAs = 2; // mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_f16_16x16x{}_f16", waveK)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
TEST_P(ABCWMMATestGFX1250, GPU_MatrixMultiplyABCF16AccWMMABFloat16)
|
|
{
|
|
const auto waveK = std::get<1>(GetParam());
|
|
const auto loadPathAB = SolutionParams::LoadPath::BufferToLDSViaVGPR;
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_F16_ACC);
|
|
|
|
if(waveK == 32)
|
|
{
|
|
REQUIRE_ARCH_CAP(GPUCapability::HasWMMA_bf16_16x16x32_bf16);
|
|
}
|
|
else
|
|
{
|
|
Throw<FatalError>("Invalid waveK value.", ShowValue(waveK));
|
|
}
|
|
matrixMultiplyABC<BFloat16, BFloat16>(16, 16, waveK, 1, loadPathAB);
|
|
|
|
const auto numWMMAs = 2; // mac_k = 2 * wave_k
|
|
const std::string wmmaMnemonic{fmt::format("v_wmma_bf16_16x16x{}_bf16", waveK)};
|
|
std::string generatedCode = m_context->instructions()->toString();
|
|
EXPECT_EQ(countSubstring(generatedCode, wmmaMnemonic), numWMMAs);
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250,
|
|
WMMATestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev0, GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(std::make_pair(rocRoller::DataType::Half, /*waveK*/ 32),
|
|
std::make_pair(rocRoller::DataType::BFloat16, /*waveK*/ 32),
|
|
std::make_pair(rocRoller::DataType::Float, /*waveK*/ 4)),
|
|
::testing::Values(std::pair<std::string, std::string>("N", "N"),
|
|
std::pair<std::string, std::string>("N", "T"),
|
|
std::pair<std::string, std::string>("T", "N"),
|
|
std::pair<std::string, std::string>("T", "T")))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250,
|
|
F16AccWMMATestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev0, GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(std::make_pair(rocRoller::DataType::Half, /*waveK*/ 32),
|
|
std::make_pair(rocRoller::DataType::BFloat16, /*waveK*/ 32)),
|
|
::testing::Values(std::pair<std::string, std::string>("N", "N"),
|
|
std::pair<std::string, std::string>("N", "T"),
|
|
std::pair<std::string, std::string>("T", "N"),
|
|
std::pair<std::string, std::string>("T", "T")))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250Rev1,
|
|
MixedWMMATestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(rocRoller::DataType::FP8, rocRoller::DataType::BF8),
|
|
::testing::Values(rocRoller::DataType::FP8, rocRoller::DataType::BF8),
|
|
::testing::Values(/*waveK*/ 128),
|
|
::testing::Values(std::pair<std::string, std::string>("N", "N"),
|
|
std::pair<std::string, std::string>("N", "T"),
|
|
std::pair<std::string, std::string>("T", "N"),
|
|
std::pair<std::string, std::string>("T", "T")))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250,
|
|
MixedWMMATestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev0, GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(rocRoller::DataType::FP8, rocRoller::DataType::BF8),
|
|
::testing::Values(rocRoller::DataType::FP8, rocRoller::DataType::BF8),
|
|
::testing::Values(/*waveK*/ 64),
|
|
::testing::Values(std::pair<std::string, std::string>("N", "N"),
|
|
std::pair<std::string, std::string>("N", "T"),
|
|
std::pair<std::string, std::string>("T", "N"),
|
|
std::pair<std::string, std::string>("T", "T")))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250,
|
|
MixedWMMAF8F6F4TestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev0, GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(::testing::Values(rocRoller::DataType::FP8,
|
|
rocRoller::DataType::BF8,
|
|
rocRoller::DataType::FP6,
|
|
rocRoller::DataType::BF6,
|
|
rocRoller::DataType::FP4),
|
|
::testing::Values(rocRoller::DataType::FP8,
|
|
rocRoller::DataType::BF8,
|
|
rocRoller::DataType::FP6,
|
|
rocRoller::DataType::BF6,
|
|
rocRoller::DataType::FP4),
|
|
::testing::Values(/*waveK*/ 128),
|
|
::testing::Values(std::pair<std::string, std::string>("N", "N"),
|
|
std::pair<std::string, std::string>("N", "T"),
|
|
std::pair<std::string, std::string>("T", "N"),
|
|
std::pair<std::string, std::string>("T", "T")))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250Rev1,
|
|
WMMAF4TestGFX1250,
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev1),
|
|
// TODO: implement transpose support for v_wmma_f32_32x16x128_f4, then re-enable other transposition cases.
|
|
::testing::Values(std::pair<std::string, std::string>("T", "N"))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250Rev1,
|
|
WMMAF4ScaledTestGFX1250,
|
|
filterValidDataTypeScaleTypeParams<WMMAF4ScaledTestGFX1250::ParamType>(::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(rocRoller::DataType::FP4),
|
|
::testing::Values(rocRoller::DataType::FP4),
|
|
::testing::Values(rocRoller::DataType::E8M0,
|
|
rocRoller::DataType::E5M3,
|
|
rocRoller::DataType::E4M3),
|
|
::testing::Values(rocRoller::DataType::E8M0,
|
|
rocRoller::DataType::E5M3,
|
|
rocRoller::DataType::E4M3),
|
|
::testing::Values(/*scaleBlockSize*/ 16, 32),
|
|
// TODO: mxDataGenerator does not work when fast-moving dim is not multiple
|
|
// of scale-block size (the case for non-TN cases).
|
|
::testing::Values(std::pair<std::string, std::string>("T", "N"))))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
MatrixMultiply1250,
|
|
MixedWMMAF8F6F4ScaledTestGFX1250,
|
|
filterValidDataTypeScaleTypeParams<MixedWMMAF8F6F4ScaledTestGFX1250::ParamType>(
|
|
::testing::Combine(
|
|
::testing::Values(GPUArchTargetGFX1250Rev0, GPUArchTargetGFX1250Rev1),
|
|
::testing::Combine(
|
|
::testing::Values(rocRoller::DataType::FP8,
|
|
rocRoller::DataType::BF8,
|
|
rocRoller::DataType::FP6,
|
|
rocRoller::DataType::BF6,
|
|
rocRoller::DataType::FP4),
|
|
::testing::Values(rocRoller::DataType::FP8,
|
|
rocRoller::DataType::BF8,
|
|
rocRoller::DataType::FP6,
|
|
rocRoller::DataType::BF6,
|
|
rocRoller::DataType::FP4),
|
|
::testing::Values(rocRoller::DataType::E8M0,
|
|
rocRoller::DataType::E5M3,
|
|
rocRoller::DataType::E4M3),
|
|
::testing::Values(rocRoller::DataType::E8M0,
|
|
rocRoller::DataType::E5M3,
|
|
rocRoller::DataType::E4M3),
|
|
::testing::Values(/*waveK*/ 128),
|
|
::testing::Values(/*scaleBlockSize*/ 16, 32),
|
|
// TODO: mxDataGenerator does not work when fast-moving dim is not multiple
|
|
// of scale-block size (the case for non-TN cases).
|
|
::testing::Values(std::pair<std::string, std::string>("T", "N"))))));
|
|
|
|
INSTANTIATE_TEST_SUITE_P(MatrixMultiplyABCWMMA1250,
|
|
ABCWMMATestGFX1250,
|
|
::testing::Combine(::testing::Values(GPUArchTargetGFX1250Rev0,
|
|
GPUArchTargetGFX1250Rev1),
|
|
::testing::Values(/*waveK*/ 32)));
|
|
} // namespace MatrixMultiplyTest
|