derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
408 lines
20 KiB
C++
408 lines
20 KiB
C++
/*******************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*
|
|
*******************************************************************************/
|
|
#ifndef GEMM_CONFIG_HPP
|
|
#define GEMM_CONFIG_HPP
|
|
|
|
#pragma GCC diagnostic push
|
|
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
|
#include <rocwmma/rocwmma.hpp>
|
|
#include <rocwmma/rocwmma_transforms.hpp>
|
|
#pragma GCC diagnostic pop
|
|
|
|
#include "gemm_driver.hpp"
|
|
#include "gemm_global_mapping.hpp"
|
|
#include "gemm_local_mapping.hpp"
|
|
|
|
#define __ROCWMMA_GEMM_LAUNCH_BOUNDS__ __launch_bounds__(Constants::AMDGCN_WAVE_SIZE * 4u)
|
|
|
|
namespace rocwmma
|
|
{
|
|
namespace CooperativeGemm
|
|
{
|
|
namespace BlockLevel
|
|
{
|
|
/* Block-Level cooperative GEMMs:
|
|
* This GEMM configuration enables collaborative data movement
|
|
* on individual blocks, or fragments. Wave collaboration depends
|
|
* on locality:
|
|
* - matrix_a collaborative waves in the same row
|
|
* - matrix_b collaborative waves in the same col
|
|
*
|
|
* Class name LDSXY indicates whether X = matrix_a or Y = matrix_b is
|
|
* transposed (T) or non-transposed (N) upon writing to LDS memory.
|
|
*
|
|
* Class name LDSRF indicates that both matrix_a and matrix_b are stored
|
|
* as a register-file (RF) in LDS memory.
|
|
*
|
|
* Due to collaboration, data is not MFMA friendly until written to LDS.
|
|
*/
|
|
struct LdsNT
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX = 0,
|
|
uint32_t TBlockY = 0>
|
|
using GlobalMapping = GlobalMapping::BlockLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingNT<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
struct LdsTN
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX = 0,
|
|
uint32_t TBlockY = 0>
|
|
using GlobalMapping = GlobalMapping::BlockLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingTN<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
struct LdsRF
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX = 0,
|
|
uint32_t TBlockY = 0>
|
|
using GlobalMapping = GlobalMapping::BlockLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingRF<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
} // BlockLevel
|
|
namespace WaveLevel
|
|
{
|
|
/* Wave-Level cooperative GEMMs:
|
|
* This GEMM configuration enables collaborative data movement
|
|
* on a collection of blocks (BlocksX x BlocksY) as a wave tile.
|
|
* Wave collaboration depends on locality:
|
|
* - matrix_a collaborative waves in the same row
|
|
* - matrix_b collaborative waves in the same col
|
|
*
|
|
* Class name LDSXY indicates whether X = matrix_a or Y = matrix_b is
|
|
* transposed (T) or non-transposed (N) upon writing to LDS memory.
|
|
*
|
|
* Due to collaboration, data is not MFMA friendly until written to LDS.
|
|
*/
|
|
struct LdsNT
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX = 0,
|
|
uint32_t TBlockY = 0>
|
|
using GlobalMapping = GlobalMapping::WaveLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingNT<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
struct LdsTN
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX = 0,
|
|
uint32_t TBlockY = 0>
|
|
using GlobalMapping = GlobalMapping::WaveLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingTN<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
} // namespace WaveLevel
|
|
|
|
namespace WorkgroupLevel
|
|
{
|
|
/* Workgroup-Level cooperative GEMMs:
|
|
* This GEMM configuration enables collaborative data movement
|
|
* on a collection of wave tiles (BlocksX x BlocksY) x (WavesX x WavesY)
|
|
* as a larger macro tile. Wave collaboration is among all waves in the
|
|
* workgroup.
|
|
*
|
|
* Class name LDSXY indicates whether X = matrix_a or Y = matrix_b is
|
|
* transposed (T) or non-transposed (N) upon writing to LDS memory.
|
|
*
|
|
* Due to collaboration, data is not MFMA friendly until written to LDS.
|
|
*/
|
|
|
|
struct LdsNT
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX,
|
|
uint32_t TBlockY>
|
|
using GlobalMapping = GlobalMapping::WorkgroupLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingNT<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
struct LdsTN
|
|
{
|
|
template <uint32_t BlockM,
|
|
uint32_t BlockN,
|
|
uint32_t BlockK,
|
|
typename InputT,
|
|
typename OutputT,
|
|
typename ComputeT,
|
|
typename LayoutA,
|
|
typename LayoutB,
|
|
typename LayoutC,
|
|
typename LayoutD,
|
|
uint32_t BlocksX,
|
|
uint32_t BlocksY,
|
|
uint32_t TBlockX,
|
|
uint32_t TBlockY>
|
|
using GlobalMapping = GlobalMapping::WorkgroupLevelMapping<BlockM,
|
|
BlockN,
|
|
BlockK,
|
|
InputT,
|
|
OutputT,
|
|
ComputeT,
|
|
LayoutA,
|
|
LayoutB,
|
|
LayoutC,
|
|
LayoutD,
|
|
BlocksX,
|
|
BlocksY,
|
|
TBlockX,
|
|
TBlockY>;
|
|
|
|
template <typename GlobalMapping, typename LayoutLds>
|
|
using LdsMapping = LocalMapping::LdsMappingTN<GlobalMapping, LayoutLds>;
|
|
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
using GemmDriver = GemmDriver<GlobalMapping, LdsMapping>;
|
|
};
|
|
|
|
} // namespace WorkgroupLevel
|
|
|
|
} // namespace CooperativeGemm
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::BlockLevel::LdsNT>()
|
|
{
|
|
return "Block_LdsNT";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::BlockLevel::LdsTN>()
|
|
{
|
|
return "Block_LdsTN";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::BlockLevel::LdsRF>()
|
|
{
|
|
return "Block_LdsRF";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::WaveLevel::LdsNT>()
|
|
{
|
|
return "Wave_LdsNT";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::WaveLevel::LdsTN>()
|
|
{
|
|
return "Wave_LdsTN";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::WorkgroupLevel::LdsNT>()
|
|
{
|
|
return "Workgroup_LdsNT";
|
|
}
|
|
|
|
template <>
|
|
constexpr const char* dataTypeToString<typename CooperativeGemm::WorkgroupLevel::LdsTN>()
|
|
{
|
|
return "Workgroup_LdsTN";
|
|
}
|
|
|
|
} // namespace rocwmma
|
|
|
|
#endif // GEMM_CONFIG_HPP
|