derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
3366 lines
244 KiB
C++
3366 lines
244 KiB
C++
/* ************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (C) 2024-2025 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
*
|
|
* ************************************************************************ */
|
|
#include "matrix_transform.h"
|
|
#include <cstdint>
|
|
#include <hip/hip_bfloat16.h>
|
|
#include <hip/hip_fp16.h>
|
|
#include <hip/hip_runtime.h>
|
|
|
|
namespace amd_detail
|
|
{
|
|
template <typename DType, size_t VectorWidth>
|
|
struct VectorIOType
|
|
{
|
|
using vtype = void;
|
|
constexpr static size_t vw{VectorWidth};
|
|
};
|
|
|
|
template <>
|
|
struct VectorIOType<int8_t, 4>
|
|
{
|
|
using vtype = char4;
|
|
};
|
|
|
|
template <>
|
|
struct VectorIOType<int32_t, 4>
|
|
{
|
|
using vtype = int4;
|
|
};
|
|
|
|
template <>
|
|
struct VectorIOType<float, 4>
|
|
{
|
|
using vtype = float4;
|
|
};
|
|
|
|
template <bool RowMaj>
|
|
__device__ uint32_t getOffset(uint32_t row, uint32_t col, uint32_t ld)
|
|
{
|
|
if constexpr(RowMaj)
|
|
{
|
|
return ld * row + col;
|
|
}
|
|
else
|
|
{
|
|
return ld * col + row;
|
|
}
|
|
}
|
|
|
|
template <bool RowMaj, uint32_t TileM, uint32_t TileN, uint32_t VectorWidth>
|
|
__device__ uint32_t getThreadLocalRowIdx(uint32_t tId)
|
|
{
|
|
constexpr auto numComponentsM
|
|
= RowMaj ? TileM : TileM / VectorWidth + !!(TileM % VectorWidth);
|
|
constexpr auto v = RowMaj ? 1 : VectorWidth;
|
|
return (tId % numComponentsM) * v;
|
|
}
|
|
|
|
template <bool RowMaj, uint32_t TileM, uint32_t TileN, uint32_t VectorWidth>
|
|
__device__ uint32_t getThreadLocalColIdx(uint32_t tId)
|
|
{
|
|
constexpr auto numComponentsM
|
|
= RowMaj ? TileM : TileM / VectorWidth + !!(TileM % VectorWidth);
|
|
constexpr auto v = RowMaj ? VectorWidth : 1;
|
|
return (tId / numComponentsM) * v;
|
|
}
|
|
|
|
template <bool RowMaj>
|
|
uint32_t getLeadingDimSize(uint32_t numRows, uint32_t numCols)
|
|
{
|
|
return RowMaj ? numCols : numRows;
|
|
}
|
|
|
|
template <typename DType,
|
|
typename ScaleType,
|
|
bool RowMajA,
|
|
bool RowMajB,
|
|
bool RowMajC,
|
|
uint32_t NumThreadsM,
|
|
uint32_t NumThreadsN,
|
|
uint32_t VectorWidth>
|
|
__device__ void transform(DType* c,
|
|
const DType* a,
|
|
const DType* b,
|
|
ScaleType alpha,
|
|
const ScaleType* alphaPtr,
|
|
ScaleType beta,
|
|
const ScaleType* betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
constexpr auto TileM = RowMajC ? NumThreadsM : NumThreadsM * VectorWidth;
|
|
constexpr auto TileN = RowMajC ? NumThreadsN * VectorWidth : NumThreadsN;
|
|
const auto tId = threadIdx.x;
|
|
const auto bId = blockIdx.x;
|
|
const auto numThreadsPerBlock = blockDim.x;
|
|
assert(TileM * TileN == numThreadsPerBlock * VectorWidth);
|
|
const auto numTilesM = numRows / TileM + !!(numRows % TileM);
|
|
const auto batchIdx = blockIdx.z;
|
|
const auto blockRow = (bId % numTilesM) * TileM;
|
|
const auto blockCol = (bId / numTilesM) * TileN;
|
|
const auto tRow = getThreadLocalRowIdx<RowMajC, TileM, TileN, VectorWidth>(tId);
|
|
const auto tCol = getThreadLocalColIdx<RowMajC, TileM, TileN, VectorWidth>(tId);
|
|
const auto row = blockRow + tRow;
|
|
const auto col = blockCol + tCol;
|
|
|
|
if(alphaPtr)
|
|
{
|
|
alpha = *alphaPtr;
|
|
}
|
|
|
|
if(betaPtr)
|
|
{
|
|
beta = *betaPtr;
|
|
}
|
|
|
|
const auto batchOffset = batchIdx * batchStride;
|
|
|
|
if constexpr(VectorWidth == 1)
|
|
{
|
|
if(row < numRows && col < numCols)
|
|
{
|
|
const auto offsetA = (transA ? getOffset<RowMajA>(col, row, ldA)
|
|
: getOffset<RowMajA>(row, col, ldA))
|
|
+ batchOffset;
|
|
const auto offsetB = (transB ? getOffset<RowMajB>(col, row, ldB)
|
|
: getOffset<RowMajB>(row, col, ldB))
|
|
+ batchOffset;
|
|
const ScaleType aData = a ? static_cast<ScaleType>(a[offsetA]) : 0;
|
|
const ScaleType bData = b ? static_cast<ScaleType>(b[offsetB]) : 0;
|
|
const DType cData = static_cast<DType>(aData * alpha + bData * beta);
|
|
const auto offsetC
|
|
= getOffset<RowMajC>(tRow + blockRow, tCol + blockCol, ldC) + batchOffset;
|
|
c[offsetC] = cData;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
const auto vectorWriteDirSize = RowMajC ? numCols : numRows;
|
|
const auto blockVectorWriteEndBound
|
|
= RowMajC ? (col + VectorWidth) : (row + VectorWidth);
|
|
const auto vectorShift = blockVectorWriteEndBound > vectorWriteDirSize
|
|
? (blockVectorWriteEndBound - vectorWriteDirSize)
|
|
: 0;
|
|
ScaleType aData[VectorWidth] = {};
|
|
ScaleType bData[VectorWidth] = {};
|
|
|
|
if constexpr(RowMajC)
|
|
{
|
|
if(row >= numRows)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
|
|
if constexpr(!RowMajC)
|
|
{
|
|
if(col >= numCols)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
|
|
#pragma unroll
|
|
for(uint32_t i = 0; i < VectorWidth; ++i)
|
|
{
|
|
uint32_t offsetA, offsetB;
|
|
|
|
if constexpr(RowMajC)
|
|
{
|
|
offsetA = (transA ? getOffset<RowMajA>(col + i - vectorShift, row, ldA)
|
|
: getOffset<RowMajA>(row, col + i - vectorShift, ldA))
|
|
+ batchOffset;
|
|
offsetB = (transB ? getOffset<RowMajB>(col + i - vectorShift, row, ldB)
|
|
: getOffset<RowMajB>(row, col + i - vectorShift, ldB))
|
|
+ batchOffset;
|
|
}
|
|
else
|
|
{
|
|
offsetA = (transA ? getOffset<RowMajA>(col, row + i - vectorShift, ldA)
|
|
: getOffset<RowMajA>(row + i - vectorShift, col, ldA))
|
|
+ batchOffset;
|
|
offsetB = (transB ? getOffset<RowMajB>(col, row + i - vectorShift, ldB)
|
|
: getOffset<RowMajB>(row + i - vectorShift, col, ldB))
|
|
+ batchOffset;
|
|
}
|
|
|
|
aData[i] = a ? static_cast<ScaleType>(a[offsetA]) : 0;
|
|
bData[i] = b ? static_cast<ScaleType>(b[offsetB]) : 0;
|
|
}
|
|
|
|
// only begin index is required, since vector write always along with
|
|
// continuous direction
|
|
uint32_t cOffset{};
|
|
|
|
if constexpr(RowMajC)
|
|
{
|
|
cOffset = getOffset<RowMajC>(tRow + blockRow, tCol + blockCol - vectorShift, ldC)
|
|
+ batchOffset;
|
|
}
|
|
else
|
|
{
|
|
cOffset = getOffset<RowMajC>(tRow + blockRow - vectorShift, tCol + blockCol, ldC)
|
|
+ batchOffset;
|
|
}
|
|
|
|
using VectorType = typename VectorIOType<DType, VectorWidth>::vtype;
|
|
|
|
if constexpr(std::is_same<VectorType, void>::value)
|
|
{
|
|
DType cData[VectorWidth];
|
|
#pragma unroll
|
|
for(uint32_t i = 0; i < VectorWidth; ++i)
|
|
{
|
|
cData[i] = static_cast<DType>(alpha * aData[i] + beta * bData[i]);
|
|
}
|
|
|
|
if(!vectorShift)
|
|
{
|
|
#pragma unroll
|
|
for(uint32_t i = 0; i < VectorWidth; ++i)
|
|
{
|
|
c[cOffset + i] = cData[i];
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for(uint32_t i = vectorShift; i < VectorWidth; ++i)
|
|
{
|
|
c[cOffset + i] = cData[i];
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
VectorType cData;
|
|
#pragma unroll
|
|
for(uint32_t i = 0; i < VectorWidth; ++i)
|
|
{
|
|
HIPBLASLT_HIPVEC_ACCESS(cData)
|
|
[i] = static_cast<DType>(alpha * aData[i] + beta * bData[i]);
|
|
}
|
|
|
|
if(!vectorShift)
|
|
{
|
|
#pragma unroll
|
|
for(uint32_t i = 0; i < VectorWidth; ++i)
|
|
{
|
|
c[cOffset + i] = HIPBLASLT_HIPVEC_ACCESS(cData)[i];
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for(uint32_t i = vectorShift; i < VectorWidth; ++i)
|
|
{
|
|
c[cOffset + i] = HIPBLASLT_HIPVEC_ACCESS(cData)[i];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} // namespace amd_detail
|
|
|
|
extern "C" {
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 1, 1, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 1, 1, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 1, 0, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 1, 0, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 0, 1, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 0, 1, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 0, 0, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 1, 0, 0, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 1, 1, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 1, 1, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 1, 0, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 1, 0, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 0, 1, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 0, 1, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 0, 0, 16, 16, 1)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(S, S, 0, 0, 0, 16, 16, 4)(DTYPE(S) * c,
|
|
const DTYPE(S) * a,
|
|
const DTYPE(S) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(S), DTYPE(S), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 1, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 1, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 1, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 1, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 0, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 0, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 0, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 1, 0, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 1, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 1, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 1, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 1, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 0, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 0, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 0, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, H, 0, 0, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(H) alpha,
|
|
const DTYPE(H) * alphaPtr,
|
|
DTYPE(H) beta,
|
|
const DTYPE(H) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(H), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 1, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 1, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 1, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 1, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 0, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 0, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 0, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 1, 0, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 1, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 1, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 1, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 1, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 0, 1, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 0, 1, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 0, 0, 16, 16, 1)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(H, S, 0, 0, 0, 16, 16, 4)(DTYPE(H) * c,
|
|
const DTYPE(H) * a,
|
|
const DTYPE(H) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(H), DTYPE(S), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 1, 1, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 1, 1, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 1, 0, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 1, 0, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 0, 1, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 0, 1, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 0, 0, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 1, 0, 0, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 1, 1, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 1, 1, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 1, 0, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 1, 0, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 0, 1, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 0, 1, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 0, 0, 16, 16, 1)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(BF16, S, 0, 0, 0, 16, 16, 4)(DTYPE(BF16) * c,
|
|
const DTYPE(BF16) * a,
|
|
const DTYPE(BF16) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(BF16), DTYPE(S), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 1, 1, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 1, 1, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 1, 0, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 1, 0, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 0, 1, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 0, 1, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 0, 0, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 1, 0, 0, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 1, 1, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 1, 1, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 1, 0, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 1, 0, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 0, 1, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 0, 1, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 0, 0, 16, 16, 1)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I8, S, 0, 0, 0, 16, 16, 4)(DTYPE(I8) * c,
|
|
const DTYPE(I8) * a,
|
|
const DTYPE(I8) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I8), DTYPE(S), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 1, 1, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 1, 1, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 1, 0, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 1, 0, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 0, 1, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 0, 1, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 0, 0, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 1, 0, 0, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), true, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 1, 1, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, true, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 1, 1, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, true, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 1, 0, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, true, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 1, 0, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, true, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 0, 1, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, false, true, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 0, 1, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, false, true, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 0, 0, 16, 16, 1)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, false, false, 16, 16, 1>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
__global__ void TRANSFORM_FUNC_NAME(I32, S, 0, 0, 0, 16, 16, 4)(DTYPE(I32) * c,
|
|
const DTYPE(I32) * a,
|
|
const DTYPE(I32) * b,
|
|
DTYPE(S) alpha,
|
|
const DTYPE(S) * alphaPtr,
|
|
DTYPE(S) beta,
|
|
const DTYPE(S) * betaPtr,
|
|
uint32_t numRows,
|
|
uint32_t numCols,
|
|
uint32_t ldA,
|
|
uint32_t ldB,
|
|
uint32_t ldC,
|
|
uint32_t batchStride,
|
|
bool transA,
|
|
bool transB)
|
|
{
|
|
amd_detail::transform<DTYPE(I32), DTYPE(S), false, false, false, 16, 16, 4>(c,
|
|
a,
|
|
b,
|
|
alpha,
|
|
alphaPtr,
|
|
beta,
|
|
betaPtr,
|
|
numRows,
|
|
numCols,
|
|
ldA,
|
|
ldB,
|
|
ldC,
|
|
batchStride,
|
|
transA,
|
|
transB);
|
|
}
|
|
}
|