derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
247 lines
12 KiB
C++
247 lines
12 KiB
C++
/*******************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*
|
|
*******************************************************************************/
|
|
#ifndef GEMM_DRIVER_HPP
|
|
#define GEMM_DRIVER_HPP
|
|
|
|
namespace rocwmma
|
|
{
|
|
/* GemmDriver class:
|
|
* This class implements a generic GEMM workflow to be used within kernel
|
|
* device code. It implements workflow steps using meta-data information
|
|
* from Global, Lds and Scheduler classes. It enables either Block-Level,
|
|
* Wave-Level or Workgroup-Level kernel workflows based on ADL and input
|
|
* fragment types.
|
|
*/
|
|
|
|
namespace CooperativeGemm
|
|
{
|
|
template <typename GlobalMapping, typename LdsMapping>
|
|
struct GemmDriver
|
|
{
|
|
// Global fragment types
|
|
using GRFragA = typename GlobalMapping::GRFragA;
|
|
using GRFragB = typename GlobalMapping::GRFragB;
|
|
|
|
// Mfma fragment types
|
|
using MfmaFragA = typename GlobalMapping::MfmaFragA;
|
|
using MfmaFragB = typename GlobalMapping::MfmaFragB;
|
|
using MfmaFragC = typename GlobalMapping::MfmaFragC;
|
|
using MfmaFragD = typename GlobalMapping::MfmaFragD;
|
|
using MfmaFragAcc = typename GlobalMapping::MfmaFragAcc;
|
|
|
|
// Local fragment types
|
|
using LWFragA = typename LdsMapping::LWFragA;
|
|
using LWFragB = typename LdsMapping::LWFragB;
|
|
using LRFragA = typename LdsMapping::LRFragA;
|
|
using LRFragB = typename LdsMapping::LRFragB;
|
|
|
|
template <typename FragT>
|
|
using MappingUtil = GetMappingUtil_t<FragT>;
|
|
|
|
template <typename FragT>
|
|
using GetIOTraitsFragA = typename GetIOConfig_t<FragT>::IOTraits;
|
|
|
|
template <typename FragT>
|
|
using GetIOTraitsFragB = typename GetIOConfig_t<FragT>::IOTraits;
|
|
|
|
// Ensure that splitCounts are the same on both sides of
|
|
// global fetch and local writes to match fragment data locality.
|
|
constexpr static auto splitCountA
|
|
= std::min((uint32_t)GetIOTraitsFragA<GRFragA>::IOCount,
|
|
(uint32_t)GetIOTraitsFragA<LWFragA>::IOCount);
|
|
|
|
constexpr static auto splitCountB
|
|
= std::min((uint32_t)GetIOTraitsFragB<GRFragB>::IOCount,
|
|
(uint32_t)GetIOTraitsFragB<LWFragB>::IOCount);
|
|
|
|
static_assert(
|
|
((uint32_t)GetIOTraitsFragA<GRFragA>::IOCount % splitCountA == 0u)
|
|
&& ((uint32_t)GetIOTraitsFragA<LWFragA>::IOCount % splitCountA == 0u),
|
|
"splitCount A is not common divisor of GlobalRead and LocalWrite IOCounts");
|
|
|
|
static_assert(
|
|
((uint32_t)GetIOTraitsFragB<GRFragB>::IOCount % splitCountB == 0u)
|
|
&& ((uint32_t)GetIOTraitsFragB<LWFragB>::IOCount % splitCountB == 0u),
|
|
"splitCount B is not common divisor of GlobalRead and LocalWrite IOCounts");
|
|
|
|
///
|
|
/// Broadcast (fill) value
|
|
///
|
|
|
|
// Broadcast value to fragment
|
|
// Single or BlocksX * BlocksY frags
|
|
template <typename FragT>
|
|
__device__ static inline void fill(FragT& frag, GetDataType_t<FragT> value);
|
|
template <typename FragT, uint32_t BlocksX, uint32_t BlocksY>
|
|
__device__ static inline void fill(FragT (&frags)[BlocksX][BlocksY],
|
|
GetDataType_t<FragT> value);
|
|
|
|
///
|
|
/// Global R/W
|
|
///
|
|
|
|
// Global A/B reads in cooperative mode
|
|
template <uint32_t BlocksX>
|
|
__device__ static inline void globalReadCoopA(GRFragA (&fragsA)[BlocksX],
|
|
GetDataType_t<GRFragA> const* gAddrA,
|
|
uint32_t lda);
|
|
__device__ static inline void globalReadCoopA(GRFragA& grFragA,
|
|
GetDataType_t<GRFragA> const* gAddrA,
|
|
uint32_t lda);
|
|
|
|
template <uint32_t BlocksY>
|
|
__device__ static inline void globalReadCoopB(GRFragB (&fragsB)[BlocksY],
|
|
GetDataType_t<GRFragB> const* gAddrB,
|
|
uint32_t ldb);
|
|
__device__ static inline void globalReadCoopB(GRFragB& grFragB,
|
|
GetDataType_t<GRFragB> const* gAddrB,
|
|
uint32_t ldb);
|
|
|
|
// Global C reads non-cooperative
|
|
// Single or BlocksX * BlocksY frags
|
|
template <uint32_t BlocksX, uint32_t BlocksY>
|
|
__device__ static inline void globalReadC(MfmaFragC (&fragC)[BlocksX][BlocksY],
|
|
GetDataType_t<MfmaFragC> const* gAddrC,
|
|
uint32_t ldc);
|
|
__device__ static inline void
|
|
globalReadC(MfmaFragC& fragC, GetDataType_t<MfmaFragC> const* gAddrC, uint32_t ldc);
|
|
|
|
// Global D writes non-cooperative
|
|
// Single or BlocksX * BlocksY frags
|
|
template <uint32_t BlocksX, uint32_t BlocksY>
|
|
__device__ static inline void globalWriteD(GetDataType_t<MfmaFragD>* gAddrD,
|
|
MfmaFragD const (&fragsD)[BlocksX][BlocksY],
|
|
uint32_t ldd);
|
|
__device__ static inline void globalWriteD(GetDataType_t<MfmaFragD>* gAddrD,
|
|
MfmaFragD const& fragD,
|
|
uint32_t ldd);
|
|
|
|
///
|
|
/// Local R/W
|
|
///
|
|
|
|
// Local A/B writes in cooperative mode
|
|
template <uint32_t BlocksX>
|
|
__device__ static inline void localWriteCoopA(GetDataType_t<GRFragA>* ldsAddr,
|
|
GRFragA const (&grFragsA)[BlocksX],
|
|
uint32_t ldlds);
|
|
__device__ static inline void localWriteCoopA(GetDataType_t<GRFragA>* ldsAddr,
|
|
GRFragA const& grFragA,
|
|
uint32_t ldlds);
|
|
|
|
template <uint32_t BlocksY>
|
|
__device__ static inline void localWriteCoopB(GetDataType_t<GRFragB>* ldsAddr,
|
|
GRFragB const (&grFragsB)[BlocksY],
|
|
uint32_t ldlds);
|
|
__device__ static inline void localWriteCoopB(GetDataType_t<GRFragB>* ldsAddr,
|
|
GRFragB const& grFragB,
|
|
uint32_t ldlds);
|
|
|
|
// Local A read non-cooperative
|
|
// Single or BlocksX frags
|
|
template <uint32_t BlocksX>
|
|
__device__ static inline void localReadA(MfmaFragA (&fragsA)[BlocksX],
|
|
GetDataType_t<MfmaFragA> const* ldsAddrA,
|
|
uint32_t ldlds);
|
|
__device__ static inline void localReadA(MfmaFragA& fragsA,
|
|
GetDataType_t<MfmaFragA> const* ldsAddrA,
|
|
uint32_t ldlds);
|
|
|
|
// Local B read non-cooperative
|
|
// Single or BlocksY frags
|
|
template <uint32_t BlocksY>
|
|
__device__ static inline void localReadB(MfmaFragB (&fragsB)[BlocksY],
|
|
GetDataType_t<MfmaFragB> const* ldsAddrB,
|
|
uint32_t ldlds);
|
|
__device__ static inline void localReadB(MfmaFragB& fragsB,
|
|
GetDataType_t<MfmaFragB> const* ldsAddrB,
|
|
uint32_t ldlds);
|
|
|
|
///
|
|
/// MFMA
|
|
///
|
|
|
|
// Performs mfma
|
|
// Single block, or BlocksX * BlocksY frags
|
|
template <uint32_t BlocksX, uint32_t BlocksY>
|
|
__device__ static inline void mfma(MfmaFragAcc (&fragAccOut)[BlocksX][BlocksY],
|
|
MfmaFragA const (&fragA)[BlocksX],
|
|
MfmaFragB const (&fragB)[BlocksY],
|
|
MfmaFragAcc const (&fragAccIn)[BlocksX][BlocksY]);
|
|
__device__ static inline void mfma(MfmaFragAcc& fragAccOut,
|
|
MfmaFragA const& fragA,
|
|
MfmaFragB const& fragB,
|
|
MfmaFragAcc const& fragAccIn);
|
|
|
|
///
|
|
/// Uniform fused multiply - add (FMA)
|
|
///
|
|
|
|
// Performs D = alpha * acc + beta * C, where alpha, beta are uniform scalars
|
|
template <uint32_t BlocksX, uint32_t BlocksY>
|
|
__device__ static inline void
|
|
uniformFma(MfmaFragD (&fragsD)[BlocksX][BlocksY],
|
|
GetDataType_t<MfmaFragAcc> alpha,
|
|
MfmaFragAcc const (&fragsAcc)[BlocksX][BlocksY],
|
|
GetDataType_t<MfmaFragAcc> beta,
|
|
MfmaFragC const (&fragsC)[BlocksX][BlocksY]);
|
|
__device__ static inline void uniformFma(MfmaFragD& fragD,
|
|
GetDataType_t<MfmaFragAcc> alpha,
|
|
MfmaFragAcc const& fragAcc,
|
|
GetDataType_t<MfmaFragAcc> beta,
|
|
MfmaFragC const& fragC);
|
|
|
|
///
|
|
/// Wave synchronization
|
|
///
|
|
__device__ static inline void syncWorkgroup();
|
|
|
|
template <int32_t priority = 0>
|
|
__device__ static inline void prioritize_wavefront();
|
|
|
|
template <int32_t mask = 0>
|
|
__device__ static inline void sched_barrier();
|
|
|
|
template <int32_t vmcnt = 0, int32_t lgkmcnt = 0>
|
|
__device__ static inline void mem_barrier();
|
|
|
|
template <int32_t vmcnt = 0>
|
|
__device__ static inline void vector_mem_barrier();
|
|
|
|
template <int32_t lgkmcnt = 0>
|
|
__device__ static inline void lds_mem_barrier();
|
|
|
|
__device__ static inline void scheduleKLoop();
|
|
};
|
|
|
|
} // namespace CooperativeGemm
|
|
|
|
} // namespace rocwmma
|
|
|
|
#include "gemm_driver_impl.hpp"
|
|
|
|
#endif // GEMM_DRIVER_HPP
|