derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
2229 lines
91 KiB
C++
2229 lines
91 KiB
C++
/*******************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (C) 2019-2024 Advanced Micro Devices, Inc. All rights reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*
|
|
*******************************************************************************/
|
|
|
|
#include <Tensile/ContractionSolution.hpp>
|
|
|
|
#include <Tensile/AMDGPU.hpp>
|
|
#include <Tensile/ContractionProblem.hpp>
|
|
#include <Tensile/Utils.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <cstddef>
|
|
#include <cstdlib>
|
|
#include <random>
|
|
|
|
#include <iomanip>
|
|
#include <regex>
|
|
|
|
namespace Tensile
|
|
{
|
|
namespace streamk
|
|
{
|
|
namespace math
|
|
{
|
|
/**
|
|
* Performs `(n + d - 1) / d`, but is robust against the case where
|
|
* `(n + d - 1)` would overflow.
|
|
*/
|
|
template <typename N, typename D>
|
|
__device__ __host__ inline constexpr N safe_ceil_div(N n, D d)
|
|
{
|
|
// Static cast to undo integral promotion.
|
|
return static_cast<N>(n / d + (n % d != 0 ? 1 : 0));
|
|
}
|
|
} // namespace math
|
|
|
|
constexpr size_t num_iters_per_cta(
|
|
size_t BLK_M, size_t BLK_N, size_t BLK_K, size_t m, size_t n, size_t k, int g)
|
|
{
|
|
return math::safe_ceil_div(math::safe_ceil_div(m, BLK_M) * math::safe_ceil_div(n, BLK_N)
|
|
* math::safe_ceil_div(k, BLK_K),
|
|
g);
|
|
}
|
|
|
|
constexpr size_t number_of_output_tiles(size_t BLK_M, size_t BLK_N, size_t m, size_t n)
|
|
{
|
|
size_t m_tiles = math::safe_ceil_div(m, BLK_M);
|
|
size_t n_tiles = math::safe_ceil_div(n, BLK_N);
|
|
return m_tiles * n_tiles;
|
|
}
|
|
|
|
constexpr size_t num_fixup_peers(size_t BLK_K, size_t k, size_t iters_per_cta)
|
|
{
|
|
return math::safe_ceil_div(math::safe_ceil_div(k, BLK_K), iters_per_cta);
|
|
}
|
|
|
|
std::tuple<double, size_t, size_t> predicted_runtime(size_t BLK_M,
|
|
size_t BLK_N,
|
|
size_t BLK_K,
|
|
size_t m,
|
|
size_t n,
|
|
size_t k,
|
|
int g,
|
|
double a,
|
|
double b,
|
|
double c,
|
|
double d)
|
|
{
|
|
size_t iters_per_cta = num_iters_per_cta(BLK_M, BLK_N, BLK_K, m, n, k, g);
|
|
size_t fixup_peers = num_fixup_peers(BLK_K, k, iters_per_cta);
|
|
|
|
return {a + (b * (fixup_peers > 1)) + (c * iters_per_cta) + (d * (fixup_peers - 1)),
|
|
iters_per_cta,
|
|
fixup_peers};
|
|
}
|
|
|
|
int best_predicted_grid_size(size_t BLK_M,
|
|
size_t BLK_N,
|
|
size_t BLK_K,
|
|
size_t m,
|
|
size_t n,
|
|
size_t k,
|
|
int grid_start,
|
|
int grid_end)
|
|
{
|
|
static const bool debug = Debug::Instance().printStreamKGridInfo();
|
|
|
|
// Fixed overhead alpha (a), fixed-size cost incurred by
|
|
// each work-group, e.g. the grid launch latency, the initial
|
|
// compulsary cache misses, the cost of writing the final output tile
|
|
// to C.
|
|
double a = 5.04 + 8.30;
|
|
|
|
// Beta (b) incorporates conditional costs of outputting temporary partial
|
|
// sums for scenarios where the number of output tiles does not quantize
|
|
// perfectly across the number of processors.
|
|
double b = 5.47;
|
|
|
|
// c represents instruction and stall workload of each MAC-iteration.
|
|
double c = 4.17;
|
|
|
|
// Delta (d) is the cost of reading and accumulating the partial sums from
|
|
// other work-groups covering the same tile.
|
|
double d = 18.59;
|
|
|
|
// std::vector<double> runtimes;
|
|
std::pair<int, double> min_grid_runtime;
|
|
min_grid_runtime.second = std::numeric_limits<double>::max();
|
|
int g = grid_start;
|
|
|
|
// Predict the number of CTAs to use between 1 and 304
|
|
for(; g <= grid_end; ++g)
|
|
{
|
|
auto [runtime, iters_per_cta, fixup_peers]
|
|
= predicted_runtime(BLK_M, BLK_N, BLK_K, m, n, k, g, a, b, c, d);
|
|
|
|
if(debug)
|
|
{
|
|
std::cout << "grid size: " << g << ", runtime: " << runtime
|
|
<< ", iters_per_cta: " << iters_per_cta
|
|
<< ", fixup_peers: " << fixup_peers << ", m: " << m << ", n: " << n
|
|
<< ", k: " << k << ", a: " << a << ", b: " << b << ", c: " << c
|
|
<< ", d: " << d << std::endl;
|
|
}
|
|
|
|
if(min_grid_runtime.second > runtime)
|
|
{
|
|
min_grid_runtime.first = g;
|
|
min_grid_runtime.second = runtime;
|
|
}
|
|
}
|
|
|
|
if(debug)
|
|
{
|
|
std::cout << "Number of Output Tiles: "
|
|
<< number_of_output_tiles(BLK_M, BLK_N, m, n) << std::endl;
|
|
std::cout << "Minimum runtime: " << min_grid_runtime.second
|
|
<< " @ grid size: " << min_grid_runtime.first << std::endl;
|
|
}
|
|
|
|
return min_grid_runtime.first;
|
|
}
|
|
} // namespace streamk
|
|
|
|
PerfModel perf;
|
|
|
|
int32_t ContractionSolution::staggerUIter(ContractionSolution::Problem const& problem,
|
|
ContractionSolution::Inputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
uint32_t sizeL = problem.boundSize(0);
|
|
|
|
// how many stride-sized clicks to stagger start offset
|
|
unsigned int staggerUIter = sizeMapping.staggerU;
|
|
|
|
// /DepthU/GSU
|
|
int unrollLoopIters = sizeL / sizeMapping.depthU / sizeMapping.globalSplitU;
|
|
|
|
unsigned int shifted = 1 << sizeMapping.staggerStrideShift;
|
|
|
|
while(staggerUIter > 1)
|
|
{
|
|
if(unrollLoopIters >= (staggerUIter * shifted))
|
|
break;
|
|
|
|
staggerUIter /= 2; // step down to smaller stagger
|
|
}
|
|
|
|
if(staggerUIter >= 1)
|
|
staggerUIter -= 1;
|
|
|
|
return staggerUIter;
|
|
}
|
|
|
|
// Return magic number. If magicShift is 0, compute and return it.
|
|
uint32_t ContractionSolution::magicNumberAlg1(uint32_t x, uint32_t* magicShift) const
|
|
{
|
|
uint64_t magicNum;
|
|
*magicShift = 33;
|
|
magicNum = (1L << *magicShift) / x + 1;
|
|
if((magicNum >> 32) != 0)
|
|
{
|
|
*magicShift = 31;
|
|
magicNum = (1L << *magicShift) / x + 1;
|
|
}
|
|
|
|
assert(magicNum >> 32 == 0); // ensure magic number fits
|
|
|
|
return static_cast<uint32_t>(magicNum);
|
|
}
|
|
|
|
uint32_t ContractionSolution::magicNumberAlg2(uint32_t d, uint32_t* magicShift) const
|
|
{
|
|
struct mu
|
|
{
|
|
unsigned M; // Magic number,
|
|
int a; // "add" indicator,
|
|
int s;
|
|
}; // and shift amount.
|
|
|
|
struct mu magu;
|
|
if(d == 0)
|
|
{
|
|
// Make dividend of 0 return 0
|
|
magu.M = 0;
|
|
magu.a = 0;
|
|
magu.s = 0;
|
|
}
|
|
else
|
|
{
|
|
// Must have 1 <= d <= 2**32-1.
|
|
int p;
|
|
unsigned nc, delta, q1, r1, q2, r2;
|
|
magu.a = 0; // Initialize "add" indicator.
|
|
nc = -1 - (-d) % d; // Unsigned arithmetic here.
|
|
p = 31; // Init. p.
|
|
q1 = 0x80000000 / nc; // Init. q1 = 2**p/nc.
|
|
r1 = 0x80000000 - q1 * nc; // Init. r1 = rem(2**p, nc).
|
|
q2 = 0x7FFFFFFF / d; // Init. q2 = (2**p - 1)/d.
|
|
r2 = 0x7FFFFFFF - q2 * d; // Init. r2 = rem(2**p - 1, d).
|
|
do
|
|
{
|
|
p = p + 1;
|
|
if(r1 >= nc - r1)
|
|
{
|
|
q1 = 2 * q1 + 1; // Update q1.
|
|
r1 = 2 * r1 - nc;
|
|
} // Update r1.
|
|
else
|
|
{
|
|
q1 = 2 * q1;
|
|
r1 = 2 * r1;
|
|
}
|
|
if(r2 + 1 >= d - r2)
|
|
{
|
|
if(q2 >= 0x7FFFFFFF)
|
|
magu.a = 1;
|
|
q2 = 2 * q2 + 1; // Update q2.
|
|
r2 = 2 * r2 + 1 - d;
|
|
} // Update r2.
|
|
else
|
|
{
|
|
if(q2 >= 0x80000000)
|
|
magu.a = 1;
|
|
q2 = 2 * q2;
|
|
r2 = 2 * r2 + 1;
|
|
}
|
|
delta = d - 1 - r2;
|
|
} while(p < 64 && (q1 < delta || (q1 == delta && r1 == 0)));
|
|
|
|
magu.M = q2 + 1; // Magic number
|
|
magu.s = p - 32; // and shift amount to return
|
|
}
|
|
|
|
*magicShift = magu.s;
|
|
const uint32_t abit = 0x80000000;
|
|
if(magu.a)
|
|
*magicShift |= abit;
|
|
|
|
// std::cout << " d=" << d << " M=" << magu.M << " a=" << magu.a << " s=" <<
|
|
// magu.s << "\n";
|
|
|
|
return magu.M;
|
|
}
|
|
|
|
uint32_t
|
|
ContractionSolution::magicNumber(int magicDivAlg, uint32_t x, uint32_t* magicShift) const
|
|
{
|
|
if(magicDivAlg == 1)
|
|
return magicNumberAlg1(x, magicShift);
|
|
else if(magicDivAlg == 2)
|
|
return magicNumberAlg2(x, magicShift);
|
|
else
|
|
throw std::runtime_error("bad magicDivAlg");
|
|
}
|
|
|
|
uint32_t ContractionSolution::smallMagicNumber(uint32_t x) const
|
|
{
|
|
uint64_t magicNum;
|
|
const int smallMagicShift = 31;
|
|
magicNum = (1L << smallMagicShift) / x + 1;
|
|
assert(magicNum >> 32 == 0); // ensure magic number fits
|
|
return static_cast<uint32_t>(magicNum);
|
|
}
|
|
|
|
std::vector<size_t> generatePackedIndicesA(ContractionSolution::Problem const& problem,
|
|
size_t packBatchDims)
|
|
{
|
|
std::vector<size_t> packedIndices;
|
|
|
|
// TODO -move packedIndices calc to problem decode.
|
|
for(auto idx = 0; idx < problem.a().dimensions(); idx++)
|
|
{
|
|
bool isSum = problem.boundIndices().end()
|
|
!= std::find_if(problem.boundIndices().begin(),
|
|
problem.boundIndices().end(),
|
|
[idx](const ContractionProblem::BoundIndex& bi) {
|
|
return bi.a == idx;
|
|
});
|
|
|
|
bool nonPackableBatch = false;
|
|
// TODO - base this check on if the batch is SetConstStrideA=0 - if so,
|
|
// don't pack
|
|
if(!(packBatchDims & 0x1))
|
|
{
|
|
nonPackableBatch = problem.batchIndices().end()
|
|
!= std::find_if(problem.batchIndices().begin(),
|
|
problem.batchIndices().end(),
|
|
[idx](const ContractionProblem::BatchIndex& bi) {
|
|
return bi.a == idx;
|
|
});
|
|
}
|
|
|
|
if(!isSum && !nonPackableBatch)
|
|
packedIndices.push_back(idx);
|
|
}
|
|
|
|
return packedIndices;
|
|
}
|
|
|
|
std::vector<size_t> generatePackedIndicesB(ContractionSolution::Problem const& problem,
|
|
size_t packBatchDims)
|
|
{
|
|
std::vector<size_t> packedIndices;
|
|
|
|
// Pack in all non-summation indices, except don't need magic number for the
|
|
// last one
|
|
for(auto idx = 0; idx < problem.b().dimensions(); idx++)
|
|
{
|
|
bool isSum = problem.boundIndices().end()
|
|
!= std::find_if(problem.boundIndices().begin(),
|
|
problem.boundIndices().end(),
|
|
[idx](const ContractionProblem::BoundIndex& bi) {
|
|
return bi.b == idx;
|
|
});
|
|
|
|
bool nonPackableBatch = false;
|
|
// TODO - base this check on if the batch is SetConstStrideB=0 - if so,
|
|
// don't pack
|
|
if(!(packBatchDims & 0x2))
|
|
{
|
|
nonPackableBatch = problem.batchIndices().end()
|
|
!= std::find_if(problem.batchIndices().begin(),
|
|
problem.batchIndices().end(),
|
|
[idx](const ContractionProblem::BatchIndex& bi) {
|
|
return bi.b == idx;
|
|
});
|
|
}
|
|
|
|
if(!isSum && !nonPackableBatch)
|
|
packedIndices.push_back(idx);
|
|
}
|
|
|
|
return packedIndices;
|
|
}
|
|
|
|
template <typename TypedInputs, bool T_Debug>
|
|
KernelInvocation
|
|
ContractionSolution::generateSingleCall(ContractionSolution::Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
//TENSILE_ASSERT_EXC(sizeMapping.workGroupMapping >= 0);
|
|
|
|
TensorDescriptor const& a = problem.a();
|
|
TensorDescriptor const& b = problem.b();
|
|
TensorDescriptor const& c = problem.c();
|
|
TensorDescriptor const& d = problem.d();
|
|
|
|
KernelInvocation rv;
|
|
|
|
rv.args = KernelArguments(T_Debug);
|
|
|
|
rv.args.reserve(1024, 128);
|
|
|
|
rv.kernelName = kernelName;
|
|
|
|
rv.workGroupSize.x = sizeMapping.workGroupSize.x * sizeMapping.workGroupSize.y
|
|
* sizeMapping.workGroupSize.z;
|
|
rv.workGroupSize.y = 1;
|
|
rv.workGroupSize.z = 1;
|
|
|
|
rv.numWorkGroups.x = 1;
|
|
rv.numWorkGroups.y = 1;
|
|
|
|
for(size_t i = 0; i < problem.freeIndicesA().size(); i++)
|
|
{
|
|
rv.numWorkGroups.x *= problem.freeSizeA(i);
|
|
}
|
|
for(size_t i = 0; i < problem.freeIndicesB().size(); i++)
|
|
{
|
|
rv.numWorkGroups.y *= problem.freeSizeB(i);
|
|
}
|
|
|
|
rv.numWorkGroups.z = 1;
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
{
|
|
if(sizeMapping.packBatchDims & 0x1)
|
|
rv.numWorkGroups.x *= problem.batchSize(i);
|
|
if(sizeMapping.packBatchDims & 0x2)
|
|
rv.numWorkGroups.y *= problem.batchSize(i);
|
|
if(!sizeMapping.packBatchDims)
|
|
rv.numWorkGroups.z *= problem.batchSize(i);
|
|
}
|
|
|
|
if(problem.transposeC01())
|
|
std::swap(rv.numWorkGroups.x, rv.numWorkGroups.y);
|
|
|
|
rv.numWorkGroups.x = CeilDivide(rv.numWorkGroups.x, sizeMapping.macroTile.x);
|
|
rv.numWorkGroups.y = CeilDivide(rv.numWorkGroups.y, sizeMapping.macroTile.y);
|
|
|
|
uint32_t problemNumGroupTiles0 = rv.numWorkGroups.x;
|
|
uint32_t problemNumGroupTiles1 = rv.numWorkGroups.y;
|
|
// used only when persistent kernel along batch
|
|
uint32_t problemNumGroupTiles2 = rv.numWorkGroups.z;
|
|
|
|
rv.numWorkGroups.y *= sizeMapping.globalSplitU;
|
|
|
|
size_t cuCount = 0;
|
|
size_t skGrid = 0;
|
|
auto tiles = problem.getNumTiles(sizeMapping);
|
|
int fullTiles = 0;
|
|
if(sizeMapping.streamK != 0 || sizeMapping.persistentKernel != 0)
|
|
{
|
|
AMDGPU const* pAMDGPU = dynamic_cast<AMDGPU const*>(&hardware);
|
|
assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0);
|
|
cuCount = pAMDGPU->computeUnitCount;
|
|
if(sizeMapping.streamK != 0)
|
|
{
|
|
skGrid = getSKGrid(problem, hardware, tiles);
|
|
rv.numWorkGroups.x = skGrid;
|
|
rv.numWorkGroups.y = 1;
|
|
rv.numWorkGroups.z = 1;
|
|
fullTiles = pAMDGPU->skFullTiles;
|
|
if(fullTiles == -1)
|
|
{
|
|
fullTiles = predictNumFullTiles(problem, hardware, tiles, skGrid);
|
|
}
|
|
}
|
|
}
|
|
|
|
if(sizeMapping.persistentKernel != 0)
|
|
{
|
|
size_t finalPKValue = sizeMapping.persistentKernel;
|
|
size_t problemGroups = rv.numWorkGroups.x * rv.numWorkGroups.y;
|
|
if(sizeMapping.persistentKernelAlongBatch)
|
|
{
|
|
problemGroups *= rv.numWorkGroups.z;
|
|
rv.numWorkGroups.z = 1;
|
|
}
|
|
|
|
if(finalPKValue == -1)
|
|
{
|
|
// 1. Get the largest pk value (ex.3)
|
|
// which can make the PK.G (ex.3*120=360) <= problemGroups (ex.433)
|
|
// 2. Scale by 5/8 (can try 0.5~1, to control the tiles-per-workgroup = 1~2)
|
|
finalPKValue = 5 * (problemGroups / cuCount) / 8;
|
|
finalPKValue = std::max(finalPKValue, (size_t)1);
|
|
//std::cout << "final persistent kernel value: " << finalPKValue << std::endl;
|
|
}
|
|
|
|
size_t persistentGroups = cuCount * finalPKValue;
|
|
rv.numWorkGroups.x = std::min(persistentGroups, problemGroups);
|
|
rv.numWorkGroups.y = 1;
|
|
}
|
|
|
|
rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x;
|
|
rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y;
|
|
rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z;
|
|
|
|
if(debugKernel)
|
|
{
|
|
rv.args.appendUnbound<unsigned int*>("debugBuffer");
|
|
}
|
|
|
|
rv.sharedMemBytes = 0;
|
|
|
|
size_t startStrideCD = problemType.useInitialStridesCD ? 0 : 1;
|
|
size_t startStrideAB = problemType.useInitialStridesAB ? 0 : 1;
|
|
|
|
auto problemSizes = problem.problemSizes();
|
|
TENSILE_ASSERT_EXC(problemSizes.size() > 0);
|
|
|
|
if(!isSourceKernel())
|
|
{
|
|
uint64_t tensor2dSizeC = c.totalAllocatedElements();
|
|
uint64_t tensor2dSizeA = (sizeMapping.packBatchDims & 0x1)
|
|
? a.totalAllocatedElements()
|
|
: problem.allocatedElementsNonBatchA();
|
|
uint64_t tensor2dSizeB = (sizeMapping.packBatchDims & 0x2)
|
|
? b.totalAllocatedElements()
|
|
: problem.allocatedElementsNonBatchB();
|
|
|
|
if(sizeMapping.preloadKernargs)
|
|
{
|
|
TENSILE_ASSERT_EXC(problemType.stridedBatched);
|
|
TENSILE_ASSERT_EXC(problemType.useBeta);
|
|
TENSILE_ASSERT_EXC(startStrideAB == 1);
|
|
|
|
rv.args.append<uint64_t>("tensor2dSizeA", tensor2dSizeA);
|
|
rv.args.append<uint64_t>("tensor2dSizeB", tensor2dSizeB);
|
|
|
|
auto aptr = inputs.a + a.offset();
|
|
auto bptr = inputs.b + b.offset();
|
|
rv.args.append<typename TypedInputs::AType const*>("a", aptr);
|
|
rv.args.append<typename TypedInputs::BType const*>("b", bptr);
|
|
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha", inputs.alpha);
|
|
if(std::is_same<typename TypedInputs::AlphaType, Half>::value && !isSourceKernel())
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha_2", inputs.alpha);
|
|
|
|
for(size_t i = startStrideAB; i < a.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideA", i), a.strides()[i]);
|
|
|
|
for(size_t i = startStrideAB; i < b.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideB", i), b.strides()[i]);
|
|
|
|
rv.args.append<uint32_t>("sizeSum", problemSizes.back());
|
|
}
|
|
|
|
// rv.args.append<uint64_t>("tensor2dSizeC", tensor2dSizeC);
|
|
|
|
if(!sizeMapping.preloadKernargs)
|
|
{
|
|
rv.args.append<uint64_t>("tensor2dSizeA", tensor2dSizeA);
|
|
rv.args.append<uint64_t>("tensor2dSizeB", tensor2dSizeB);
|
|
}
|
|
}
|
|
|
|
if(sizeMapping.globalAccumulation && sizeMapping.streamK == 0)
|
|
{
|
|
rv.args.append<void const*>("ws_d", inputs.ws);
|
|
rv.args.append<void const*>("ws_c", inputs.ws);
|
|
}
|
|
else if(problemType.stridedBatched)
|
|
{
|
|
auto dptr = inputs.d + d.offset();
|
|
auto cptr = inputs.c + c.offset();
|
|
rv.args.append<typename TypedInputs::DType const*>("d", dptr);
|
|
rv.args.append<typename TypedInputs::CType const*>("c", cptr);
|
|
}
|
|
else
|
|
{
|
|
rv.args.append<typename TypedInputs::DType const* const*>("batchD", inputs.batchD);
|
|
rv.args.append<typename TypedInputs::CType const* const*>("batchC", inputs.batchC);
|
|
}
|
|
|
|
if(problemType.stridedBatched)
|
|
{
|
|
if(!sizeMapping.preloadKernargs)
|
|
{
|
|
auto aptr = inputs.a + a.offset();
|
|
auto bptr = inputs.b + b.offset();
|
|
rv.args.append<typename TypedInputs::AType const*>("a", aptr);
|
|
rv.args.append<typename TypedInputs::BType const*>("b", bptr);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
rv.args.append<typename TypedInputs::AType const* const*>("batchA", inputs.batchA);
|
|
rv.args.append<typename TypedInputs::BType const* const*>("batchB", inputs.batchB);
|
|
}
|
|
|
|
if(sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 0)
|
|
{
|
|
// StreamK workspace + flags
|
|
rv.args.append<void const*>("ws", inputs.ws);
|
|
void* ws = inputs.ws;
|
|
size_t flagsOffset = partialTileSize(skGrid);
|
|
void* flags = (void*)(static_cast<char*>(ws) + flagsOffset);
|
|
rv.args.append<void*>("Flags", flags);
|
|
}
|
|
|
|
if(!problemType.stridedBatched)
|
|
{
|
|
rv.args.append<uint64_t>("offsetD", d.offset());
|
|
rv.args.append<uint64_t>("offsetC", c.offset());
|
|
rv.args.append<uint64_t>("offsetA", a.offset());
|
|
rv.args.append<uint64_t>("offsetB", b.offset());
|
|
}
|
|
|
|
if(!sizeMapping.preloadKernargs)
|
|
{
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha", inputs.alpha);
|
|
if(std::is_same<typename TypedInputs::AlphaType, Half>::value && !isSourceKernel())
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha_2", inputs.alpha);
|
|
}
|
|
|
|
if(problemType.useBeta)
|
|
{
|
|
rv.args.append<typename TypedInputs::BetaType>("beta", inputs.beta);
|
|
if(std::is_same<typename TypedInputs::BetaType, Half>::value && !isSourceKernel())
|
|
rv.args.append<typename TypedInputs::BetaType>("beta_2", inputs.beta);
|
|
}
|
|
|
|
if(sizeMapping.globalAccumulation && sizeMapping.streamK == 0)
|
|
{
|
|
size_t wsStride = startStrideCD ? d.sizes()[0] : 1;
|
|
for(size_t i = startStrideCD; i < d.dimensions(); i++)
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideW_D", i), wsStride);
|
|
wsStride *= d.sizes()[i];
|
|
}
|
|
|
|
wsStride = startStrideCD ? d.sizes()[0] : 1;
|
|
for(size_t i = startStrideCD; i < c.dimensions(); i++)
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideW_C", i), wsStride);
|
|
wsStride *= d.sizes()[i];
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for(size_t i = startStrideCD; i < d.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideD", i), d.strides()[i]);
|
|
|
|
for(size_t i = startStrideCD; i < c.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideC", i), c.strides()[i]);
|
|
}
|
|
|
|
if(!sizeMapping.preloadKernargs)
|
|
{
|
|
for(size_t i = startStrideAB; i < a.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideA", i), a.strides()[i]);
|
|
|
|
for(size_t i = startStrideAB; i < b.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideB", i), b.strides()[i]);
|
|
}
|
|
|
|
{
|
|
int maxIdx = problemSizes.size();
|
|
if(sizeMapping.preloadKernargs)
|
|
maxIdx--;
|
|
for(int idx = 0; idx < maxIdx; idx++)
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("size_", idx), problemSizes[idx]);
|
|
}
|
|
}
|
|
|
|
if(sizeMapping.packSummationDims)
|
|
// boundIndices are ordered with unroll last.
|
|
// Magic numbers for all but first are needed to unpack other dims.
|
|
for(auto si = 1; si < problem.boundIndices().size(); si++)
|
|
{
|
|
auto numIter = problem.boundSize(si);
|
|
bool isUnroll = si == problem.boundIndices().size() - 1;
|
|
if(isUnroll)
|
|
{
|
|
numIter = numIter / sizeMapping.depthU / sizeMapping.globalSplitU
|
|
* sizeMapping.depthU;
|
|
}
|
|
uint32_t magicShift;
|
|
rv.args.append<uint32_t>(
|
|
concatenate_if<T_Debug>("magicNumberNumIter_", si),
|
|
magicNumber(sizeMapping.magicDivAlg, numIter, &magicShift));
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("magicShiftNumIter_", si),
|
|
magicShift);
|
|
|
|
if(isUnroll and sizeMapping.globalSplitU > 1)
|
|
{
|
|
// compute magic number for gsu remainder iterations:
|
|
// Kernel will select whether to use above or remainder portion based on work-group assignment
|
|
rv.args.append<uint32_t>(
|
|
concatenate_if<T_Debug>("magicNumberNumIter_GsuRemainder"),
|
|
magicNumber(
|
|
sizeMapping.magicDivAlg, numIter + sizeMapping.depthU, &magicShift));
|
|
rv.args.append<uint32_t>(
|
|
concatenate_if<T_Debug>("magicShiftNumIter_GsuRemainder"), magicShift);
|
|
}
|
|
}
|
|
|
|
if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesA(problem, sizeMapping.packBatchDims);
|
|
|
|
// Pack in all non-summation indices, except don't need magic number for the
|
|
// last one
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end() - 1; pi++)
|
|
{
|
|
auto idx = *pi;
|
|
auto size = a.sizes()[idx];
|
|
uint32_t magicShift;
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("magicNumberSizeA_", idx),
|
|
magicNumber(sizeMapping.magicDivAlg, size, &magicShift));
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("magicShiftSizeA_", idx),
|
|
magicShift);
|
|
}
|
|
}
|
|
if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesB(problem, sizeMapping.packBatchDims);
|
|
|
|
// Pack in all non-summation indices, except don't need magic number for the
|
|
// last one
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end() - 1; pi++)
|
|
{
|
|
auto idx = *pi;
|
|
auto size = b.sizes()[idx];
|
|
uint32_t magicShift;
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("magicNumberSizeB_", idx),
|
|
magicNumber(sizeMapping.magicDivAlg, size, &magicShift));
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("magicShiftSizeB_", idx),
|
|
magicShift);
|
|
}
|
|
}
|
|
|
|
for(auto si : problem.boundIndices())
|
|
{
|
|
if(si.aZeroPad.valid())
|
|
{
|
|
rv.args.append<int32_t>(concatenate_if<T_Debug>("padStartA_", si.a),
|
|
si.aZeroPad.padStart);
|
|
rv.args.append<int32_t>(concatenate_if<T_Debug>("padEndA_", si.a),
|
|
si.aZeroPad.padEnd);
|
|
}
|
|
if(si.bZeroPad.valid())
|
|
{
|
|
rv.args.append<int32_t>(concatenate_if<T_Debug>("padStartB_", si.b),
|
|
si.bZeroPad.padStart);
|
|
rv.args.append<int32_t>(concatenate_if<T_Debug>("padEndB_", si.b),
|
|
si.bZeroPad.padEnd);
|
|
}
|
|
}
|
|
|
|
if(sizeMapping.staggerU)
|
|
rv.args.append<int32_t>("staggerUIter", staggerUIter(problem, inputs, hardware));
|
|
|
|
rv.args.append<uint32_t>("problemNumGroupTiles0", problemNumGroupTiles0);
|
|
rv.args.append<uint32_t>("problemNumGroupTiles1", problemNumGroupTiles1);
|
|
|
|
if(!isSourceKernel())
|
|
{
|
|
uint32_t numFullBlocks = problemNumGroupTiles1;
|
|
uint32_t wgmRemainder1 = 0;
|
|
uint32_t magicNumberWgmRemainder1 = 0;
|
|
|
|
// conditional args, aligned with KernelWriterAssembly.py
|
|
if(sizeMapping.persistentKernel != 0 || sizeMapping.streamK != 0)
|
|
{
|
|
uint32_t magicShift;
|
|
rv.args.append<uint32_t>("magicNumberProblemNumGroupTiles0",
|
|
magicNumber(2, problemNumGroupTiles0, &magicShift));
|
|
rv.args.append<uint32_t>("magicShiftProblemNumGroupTiles0", magicShift);
|
|
}
|
|
|
|
if(sizeMapping.persistentKernel != 0)
|
|
{
|
|
rv.args.append<uint32_t>("gridNumWorkGroups0", rv.numWorkGroups.x);
|
|
}
|
|
|
|
if(sizeMapping.persistentKernelAlongBatch)
|
|
{
|
|
uint32_t numGroupTiles0x1 = problemNumGroupTiles0 * problemNumGroupTiles1;
|
|
uint32_t magicShift;
|
|
|
|
rv.args.append<uint32_t>("problemNumGroupTiles2", problemNumGroupTiles2);
|
|
rv.args.append<uint32_t>("magicNumberProblemNumGroupTiles0By1",
|
|
magicNumber(2, numGroupTiles0x1, &magicShift));
|
|
rv.args.append<uint32_t>("magicShiftProblemNumGroupTiles0By1", magicShift);
|
|
}
|
|
|
|
if(sizeMapping.streamK != 0)
|
|
{
|
|
auto itersPerTile = problem.getItersPerTile(sizeMapping);
|
|
auto totalIters = tiles * itersPerTile;
|
|
uint32_t magicNumberItersPerTile;
|
|
uint32_t magicShiftItersPerTile;
|
|
magicNumberItersPerTile = magicNumber(2, itersPerTile, &magicShiftItersPerTile);
|
|
|
|
rv.args.append<uint32_t>("itersPerTile", itersPerTile);
|
|
rv.args.append<uint32_t>("magicNumberItersPerTile", magicNumberItersPerTile);
|
|
rv.args.append<uint32_t>("magicShiftItersPerTile", magicShiftItersPerTile);
|
|
|
|
uint32_t numGroupTiles0x1 = problemNumGroupTiles0 * problemNumGroupTiles1;
|
|
uint32_t magicNumProblemNumGroupTiles0By1;
|
|
uint32_t magicShiftProblemNumGroupTiles0By1;
|
|
magicNumProblemNumGroupTiles0By1
|
|
= magicNumber(2, numGroupTiles0x1, &magicShiftProblemNumGroupTiles0By1);
|
|
rv.args.append<uint32_t>("magicNumProblemNumGroupTiles0By1",
|
|
magicNumProblemNumGroupTiles0By1);
|
|
rv.args.append<uint32_t>("magicShiftProblemNumGroupTiles0By1",
|
|
magicShiftProblemNumGroupTiles0By1);
|
|
|
|
rv.args.append<uint32_t>("totalIters", totalIters);
|
|
if(sizeMapping.streamK == 1) // Basic SK
|
|
{
|
|
uint32_t itersPerWave = CeilDivide(totalIters, rv.numWorkGroups.x);
|
|
rv.args.append<uint32_t>("SKItersPerWG", itersPerWave);
|
|
}
|
|
else if(sizeMapping.streamK >= 2) // Two-tile SK
|
|
{
|
|
// skTiles is number of Stream-K tiles to complete
|
|
// Two-tile algorithm causes each WG to run an even number of Stream-K iterations,
|
|
// followed by an even number of data-parllel tiles.
|
|
// If total tiles is evenly divisble by grid size,
|
|
// then no Stream-K tiles are needed, all data-parallel
|
|
uint32_t skTiles = skGrid;
|
|
if(tiles % skGrid != 0)
|
|
{
|
|
// Number of SK tiles determined by performance prediction
|
|
// Either runs remainder tiles, or remainder + 1 full grid of tiles
|
|
// Can be overridden by setting TENSILE_STREAMK_FULL_TILES
|
|
skTiles = std::min(skGrid * fullTiles + tiles % skGrid, tiles);
|
|
}
|
|
|
|
uint32_t skItersPerWG = skTiles * itersPerTile / skGrid;
|
|
uint32_t skExtraIters = skTiles * itersPerTile % (skGrid);
|
|
|
|
rv.args.append<uint32_t>("SKItersPerWG", skItersPerWG);
|
|
rv.args.append<uint32_t>("skGrid", skGrid);
|
|
rv.args.append<uint32_t>("skTiles", skTiles);
|
|
rv.args.append<uint32_t>("skExtraIters", skExtraIters);
|
|
}
|
|
}
|
|
|
|
if(sizeMapping.workGroupMapping != 0)
|
|
{
|
|
if(sizeMapping.workGroupMapping >= 0)
|
|
{
|
|
numFullBlocks = problemNumGroupTiles1 / sizeMapping.workGroupMapping;
|
|
wgmRemainder1 = problemNumGroupTiles1 % sizeMapping.workGroupMapping;
|
|
}
|
|
else
|
|
{
|
|
numFullBlocks = problemNumGroupTiles0 / (-sizeMapping.workGroupMapping);
|
|
wgmRemainder1 = problemNumGroupTiles0 % (-sizeMapping.workGroupMapping);
|
|
}
|
|
if(wgmRemainder1 == 0)
|
|
wgmRemainder1 = sizeMapping.workGroupMapping;
|
|
magicNumberWgmRemainder1 = smallMagicNumber(wgmRemainder1);
|
|
}
|
|
|
|
if(std::abs(sizeMapping.workGroupMapping) > 1)
|
|
{
|
|
rv.args.append<uint32_t>("numFullBlocks", numFullBlocks);
|
|
rv.args.append<uint32_t>("wgmRemainder1", wgmRemainder1);
|
|
rv.args.append<uint32_t>("magicNumberWgmRemainder1", magicNumberWgmRemainder1);
|
|
}
|
|
}
|
|
|
|
if(problemType.stochasticRounding)
|
|
{
|
|
// generate seed from random generator
|
|
std::random_device rd;
|
|
std::mt19937 gen(rd());
|
|
std::uniform_int_distribution<uint32_t> distribution(0, 0xFFFFFFFF);
|
|
uint32_t seed = distribution(gen);
|
|
//rv.args.append<uint32_t>("RNDSeed", 0x0);
|
|
rv.args.append<uint32_t>("RNDSeed", seed);
|
|
}
|
|
|
|
if(!isSourceKernel())
|
|
{
|
|
rv.args.append<uint32_t>("pad", 0);
|
|
}
|
|
|
|
rv.codeObjectFile = codeObjectFilename.load();
|
|
|
|
return rv;
|
|
}
|
|
|
|
bool ContractionSolution::isSourceKernel() const
|
|
{
|
|
return sizeMapping.sourceKernel;
|
|
}
|
|
|
|
template <typename TypedInputs, bool T_Debug>
|
|
KernelInvocation ContractionSolution::generateStreamKInitCall(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
TensorDescriptor const& c = problem.c();
|
|
TensorDescriptor const& d = problem.d();
|
|
|
|
KernelInvocation rv;
|
|
|
|
rv.args = KernelArguments(T_Debug);
|
|
|
|
rv.args.reserve(512, 64);
|
|
|
|
rv.kernelName = streamKInitKernelName(problem, inputs, hardware);
|
|
|
|
rv.workGroupSize.x = 256;
|
|
rv.workGroupSize.y = 1;
|
|
rv.workGroupSize.z = 1;
|
|
|
|
auto tiles = problem.getNumTiles(sizeMapping);
|
|
size_t skGrid = getSKGrid(problem, hardware, tiles);
|
|
size_t wiZ = 1;
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
wiZ *= problem.batchSize(i);
|
|
size_t flagCount = skGrid * wiZ;
|
|
|
|
rv.numWorkGroups.x = CeilDivide(flagCount, rv.workGroupSize.x);
|
|
rv.numWorkGroups.y = 1;
|
|
rv.numWorkGroups.z = 1;
|
|
|
|
rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x;
|
|
rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y;
|
|
rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z;
|
|
|
|
void* ws = inputs.ws;
|
|
size_t flagsOffset = partialTileSize(skGrid);
|
|
void* flags = (void*)(static_cast<char*>(ws) + flagsOffset);
|
|
rv.args.append<void*>("Flags", flags);
|
|
|
|
rv.args.append<uint32_t>("flagCount", flagCount);
|
|
|
|
//Pass along code object dependency
|
|
// TODO check this
|
|
rv.codeObjectFile = codeObjectFilename.load();
|
|
|
|
return rv;
|
|
}
|
|
|
|
template <typename TypedInputs>
|
|
std::string ContractionSolution::streamKInitKernelName(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
std::string name = "WSFlags";
|
|
return name;
|
|
}
|
|
|
|
template <typename TypedInputs, bool T_Debug>
|
|
KernelInvocation ContractionSolution::generateBetaOnlyCall(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
TensorDescriptor const& c = problem.c();
|
|
TensorDescriptor const& d = problem.d();
|
|
|
|
KernelInvocation rv;
|
|
|
|
rv.args = KernelArguments(T_Debug);
|
|
|
|
rv.args.reserve(512, 64);
|
|
|
|
rv.kernelName = betaOnlyKernelName(problem, inputs, hardware);
|
|
|
|
rv.workGroupSize.x = 256;
|
|
rv.workGroupSize.y = 1;
|
|
rv.workGroupSize.z = 1;
|
|
|
|
size_t wiX = 1;
|
|
size_t wiY = 1;
|
|
size_t wiZ = 1;
|
|
for(size_t i = 0; i < problem.freeIndicesA().size(); i++)
|
|
wiX *= problem.freeSizeA(i);
|
|
for(size_t i = 0; i < problem.freeIndicesB().size(); i++)
|
|
wiY *= problem.freeSizeB(i);
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
wiZ *= problem.batchSize(i);
|
|
|
|
rv.numWorkGroups.x = CeilDivide(wiX * wiY * wiZ, rv.workGroupSize.x);
|
|
rv.numWorkGroups.y = 1;
|
|
rv.numWorkGroups.z = 1;
|
|
|
|
rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x;
|
|
rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y;
|
|
rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z;
|
|
|
|
if(sizeMapping.globalAccumulation)
|
|
rv.args.append<void*>("WS", inputs.ws);
|
|
else if(problemType.stridedBatched)
|
|
{
|
|
auto dptr = inputs.d;
|
|
dptr += d.offset();
|
|
rv.args.append<typename TypedInputs::DType*>("D", dptr);
|
|
}
|
|
else
|
|
rv.args.append<typename TypedInputs::DType const* const*>("batchD", inputs.batchD);
|
|
|
|
if(problemType.stridedBatched)
|
|
{
|
|
auto cptr = inputs.c;
|
|
cptr += c.offset();
|
|
rv.args.append<typename TypedInputs::CType const*>("C", cptr);
|
|
}
|
|
else
|
|
rv.args.append<typename TypedInputs::CType const* const*>("batchC", inputs.batchC);
|
|
|
|
if(!problemType.stridedBatched)
|
|
{
|
|
rv.args.append<uint64_t>("offsetD", d.offset());
|
|
rv.args.append<uint64_t>("offsetC", c.offset());
|
|
}
|
|
|
|
if(sizeMapping.globalAccumulation)
|
|
{
|
|
size_t stride = d.sizes()[0];
|
|
for(size_t i = 1; i < d.dimensions(); i++)
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideW", i),
|
|
d.sizes()[i] == 1 ? 0 : stride);
|
|
stride *= d.sizes()[i];
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for(size_t i = 1; i < d.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideD", i),
|
|
d.sizes()[i] == 1 ? 0 : d.strides()[i]);
|
|
}
|
|
|
|
for(size_t i = 1; i < c.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideC", i),
|
|
c.sizes()[i] == 1 ? 0 : c.strides()[i]);
|
|
|
|
int idx = 0;
|
|
for(auto size : problem.d().sizes())
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("size_", idx), size);
|
|
idx++;
|
|
}
|
|
|
|
rv.args.append<typename TypedInputs::BetaType>("beta", inputs.beta);
|
|
|
|
//Pass along code object dependency
|
|
rv.codeObjectFile = codeObjectFilename.load();
|
|
|
|
return rv;
|
|
}
|
|
|
|
template <typename TypedInputs>
|
|
std::string ContractionSolution::betaOnlyKernelName(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
std::string name = concatenate(
|
|
"C", problem.cNames(), "_", TypeInfo<typename TypedInputs::DType>::Abbrev());
|
|
|
|
if(!problemType.stridedBatched)
|
|
{
|
|
name += "_GB";
|
|
}
|
|
|
|
if(sizeMapping.globalAccumulation)
|
|
{
|
|
name += "_GA";
|
|
}
|
|
|
|
return name;
|
|
}
|
|
|
|
template <typename TypedInputs, bool T_Debug>
|
|
KernelInvocation ContractionSolution::generateOutputConversionCall(
|
|
Problem const& problem, TypedInputs const& inputs, Hardware const& hardware) const
|
|
{
|
|
AMDGPU const* pAMDGPU = dynamic_cast<AMDGPU const*>(&hardware);
|
|
assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0);
|
|
size_t cuCount = pAMDGPU->computeUnitCount;
|
|
|
|
TensorDescriptor const& c = problem.c();
|
|
TensorDescriptor const& d = problem.d();
|
|
|
|
KernelInvocation rv;
|
|
|
|
rv.args = KernelArguments(T_Debug);
|
|
|
|
rv.args.reserve(512, 64);
|
|
|
|
rv.workGroupSize.x = 256;
|
|
rv.workGroupSize.y = 1;
|
|
rv.workGroupSize.z = 1;
|
|
|
|
size_t wiX = 1;
|
|
size_t wiY = 1;
|
|
size_t wiZ = 1;
|
|
for(size_t i = 0; i < problem.freeIndicesA().size(); i++)
|
|
wiX *= problem.freeSizeA(i);
|
|
for(size_t i = 0; i < problem.freeIndicesB().size(); i++)
|
|
wiY *= problem.freeSizeB(i);
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
wiZ *= problem.batchSize(i);
|
|
|
|
const unsigned int numThreadsPerCU = 256;
|
|
unsigned int gsu = static_cast<int>(sizeMapping.globalSplitU);
|
|
if(sizeMapping.globalAccumulation == 1)
|
|
// globalAccumulation = 1 case, ignore globalSplitU and use 1
|
|
gsu = 1;
|
|
// wider global load for postGSU
|
|
// only for compute type = Float
|
|
bool supportedTypeForVWopt
|
|
= problem.alphaType() == DataType::Float || problem.alphaType() == DataType::Double;
|
|
size_t total = wiX * wiY * wiZ;
|
|
int vw = 1;
|
|
size_t threshVW2
|
|
= cuCount * numThreadsPerCU * 2; // should be more than number of physical threads * vw
|
|
if(supportedTypeForVWopt && total > threshVW2 && problem.freeSizeA(0) % 2 == 0)
|
|
vw = 2;
|
|
|
|
// parallel reduction width calculation
|
|
// only for compute type = Float
|
|
bool supportedTypeForReductionOpt
|
|
= problem.alphaType() == DataType::Float || problem.alphaType() == DataType::Double;
|
|
size_t threshReduction
|
|
= cuCount
|
|
* numThreadsPerCU; // should be less than number of physical threads / reduction
|
|
const unsigned int maxReductionConst = 4;
|
|
const unsigned int minGSUperReduction
|
|
= 32; // Minimum GSU=128 for Reduction=4, GSU=64 for Reduction2
|
|
unsigned int maxReduction = std::min(maxReductionConst, gsu / minGSUperReduction);
|
|
unsigned int reduction = 1;
|
|
if(supportedTypeForReductionOpt && (gsu & (gsu - 1)) == 0 && maxReduction > 1)
|
|
{
|
|
// apply reduction only if float compute type and gsu is power of 2 (for small array only)
|
|
reduction = maxReduction;
|
|
while(reduction > 1)
|
|
{
|
|
size_t totalThread = total * reduction / vw;
|
|
if(gsu % reduction == 0 && totalThread <= threshReduction)
|
|
// found an applicable reduction
|
|
break;
|
|
// for next loop
|
|
reduction /= 2;
|
|
}
|
|
if(reduction == 2 && problem.alphaType() == DataType::Float)
|
|
// so far, reduction=2 does not perform well
|
|
reduction = 1;
|
|
}
|
|
|
|
// GSU loop unroll opt
|
|
// only for compute type = Float, Double
|
|
bool supportedTypeForUnrollOpt
|
|
= problem.alphaType() == DataType::Float || problem.alphaType() == DataType::Double;
|
|
int gsuUnrollUnit = 16 * reduction; // must match Tensile generator code
|
|
if(supportedTypeForUnrollOpt == false || sizeMapping.globalAccumulation == 1)
|
|
gsuUnrollUnit = 1;
|
|
|
|
rv.kernelName = outputConversionKernelName(
|
|
problem, inputs, gsu, vw, reduction, gsuUnrollUnit, hardware);
|
|
|
|
rv.numWorkGroups.x = CeilDivide(wiX * wiY * wiZ * reduction, rv.workGroupSize.x * vw);
|
|
rv.numWorkGroups.y = 1;
|
|
rv.numWorkGroups.z = 1;
|
|
|
|
rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x;
|
|
rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y;
|
|
rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z;
|
|
|
|
if(problemType.stridedBatched)
|
|
{
|
|
auto dptr = inputs.d;
|
|
dptr += d.offset();
|
|
rv.args.append<typename TypedInputs::DType*>("D", dptr);
|
|
}
|
|
else
|
|
rv.args.append<typename TypedInputs::DType const* const*>("batchD", inputs.batchD);
|
|
|
|
rv.args.append<void*>("WS", inputs.ws);
|
|
|
|
if(problemType.stridedBatched)
|
|
{
|
|
auto cptr = inputs.c;
|
|
cptr += c.offset();
|
|
rv.args.append<typename TypedInputs::CType const*>("C", cptr);
|
|
}
|
|
else
|
|
rv.args.append<typename TypedInputs::CType const* const*>("batchC", inputs.batchC);
|
|
|
|
if(!problemType.stridedBatched)
|
|
{
|
|
rv.args.append<uint64_t>("offsetD", d.offset());
|
|
rv.args.append<uint64_t>("offsetC", c.offset());
|
|
}
|
|
|
|
if(sizeMapping.globalAccumulation == 2)
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha", inputs.alpha);
|
|
else
|
|
rv.args.append<typename TypedInputs::AlphaType>("alpha", 1.0f);
|
|
|
|
if(sizeMapping.globalAccumulation == 2 and problemType.useBeta)
|
|
rv.args.append<typename TypedInputs::BetaType>("beta", inputs.beta);
|
|
else
|
|
rv.args.append<typename TypedInputs::BetaType>("beta", 0.0f);
|
|
|
|
for(size_t i = 1; i < d.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideD", i), d.strides()[i]);
|
|
|
|
uint32_t wsStride = d.sizes()[0];
|
|
for(size_t i = 1; i < d.dimensions(); i++)
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideW", i), wsStride);
|
|
wsStride *= d.sizes()[i];
|
|
}
|
|
|
|
for(size_t i = 1; i < c.dimensions(); i++)
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("strideC", i), c.strides()[i]);
|
|
|
|
int idx = 0;
|
|
for(auto size : problem.d().sizes())
|
|
{
|
|
rv.args.append<uint32_t>(concatenate_if<T_Debug>("size_", idx), size);
|
|
idx++;
|
|
}
|
|
|
|
rv.args.append<uint32_t>("gsu", gsu);
|
|
|
|
if(problemType.stochasticRounding)
|
|
{
|
|
// generate seed from random generator
|
|
std::random_device rd;
|
|
std::mt19937 gen(rd());
|
|
std::uniform_int_distribution<uint32_t> distribution(0, 0xFFFFFFFF);
|
|
uint32_t seed = distribution(gen);
|
|
//rv.args.append<uint32_t>("RNDSeed", 0x0);
|
|
rv.args.append<uint32_t>("RNDSeed", seed);
|
|
}
|
|
|
|
//@TODO determine if this is needed, may not end up in the same code object file
|
|
rv.codeObjectFile = codeObjectFilename.load();
|
|
|
|
return rv;
|
|
}
|
|
|
|
bool ContractionSolution::canSolve(Problem const& problem, Hardware const& hardware) const
|
|
{
|
|
static const bool debug = Debug::Instance().printPredicateEvaluation();
|
|
|
|
Task task(hardware, problem, *this);
|
|
if(debug)
|
|
{
|
|
hardwarePredicate->debugEval(hardware, std::cout);
|
|
problemPredicate->debugEval(problem, std::cout);
|
|
taskPredicate->debugEval(task, std::cout);
|
|
}
|
|
return (*taskPredicate)(task) && (*problemPredicate)(problem)
|
|
&& (*hardwarePredicate)(hardware);
|
|
}
|
|
|
|
bool ContractionSolution::matchesProblemType(Problem const& problem,
|
|
Hardware const& hardware) const
|
|
{
|
|
bool matchesHardware = (*hardwarePredicate)(hardware);
|
|
bool matchesType
|
|
= (problemType.aType == problem.a().dataType())
|
|
&& (problemType.bType == problem.b().dataType())
|
|
&& (problemType.cType == problem.c().dataType())
|
|
&& (problemType.dType == problem.d().dataType())
|
|
&& (problemType.highPrecisionAccumulate == problem.highPrecisionAccumulate())
|
|
&& (problemType.stridedBatched == problem.stridedBatched())
|
|
&& (problemType.fp16AltImpl == problem.fp16AltImpl())
|
|
&& (problemType.fp16AltImplRound == problem.fp16AltImplRound());
|
|
return matchesHardware && matchesType;
|
|
}
|
|
|
|
template <typename TypedInputs>
|
|
std::string ContractionSolution::outputConversionKernelName(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
int gsu,
|
|
int vw,
|
|
int reduction,
|
|
int gsuUnrollUnit,
|
|
Hardware const& hardware) const
|
|
{
|
|
std::string name = concatenate(
|
|
"C", problem.cNames(), "_", TypeInfo<typename TypedInputs::DType>::Abbrev());
|
|
|
|
if(!problemType.stridedBatched)
|
|
{
|
|
name += "_GB";
|
|
}
|
|
|
|
name += "_PostGSU";
|
|
|
|
// add extra string for gsu
|
|
// This part must match tensile code generation (in KernelWriterConversion.py)
|
|
// add mod related str (only for gsuUnrollUnit > 1)
|
|
if(gsuUnrollUnit > 1)
|
|
{
|
|
size_t gsuMod = gsu % gsuUnrollUnit;
|
|
if(gsuMod == 0)
|
|
gsuMod = gsuUnrollUnit;
|
|
std::string modStr = "";
|
|
if(gsu > gsuUnrollUnit)
|
|
{
|
|
modStr = "_mod" + std::to_string(gsuUnrollUnit);
|
|
}
|
|
name += std::to_string(gsuMod) + modStr;
|
|
}
|
|
if(vw >= 2)
|
|
name += "_VW" + std::to_string(vw);
|
|
|
|
if(reduction >= 2)
|
|
name += "_R" + std::to_string(reduction);
|
|
|
|
return name;
|
|
}
|
|
|
|
template <typename TypedInputs>
|
|
std::vector<KernelInvocation> ContractionSolution::solveTyped(Problem const& problem,
|
|
TypedInputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
bool debug = Debug::Instance().printKernelArguments() || this->kernelArgsLog;
|
|
bool debugSkipInit = Debug::Instance().skipInitKernelLaunch();
|
|
|
|
int boundSize = 1;
|
|
for(size_t i = 0; i < problem.boundIndices().size(); i++)
|
|
boundSize *= problem.boundSize(i);
|
|
|
|
// Check for nullptrs if alpha is non-zero.
|
|
if(((inputs.alpha != static_cast<typename TypedInputs::AlphaType>(0)) && (boundSize != 0))
|
|
&& ((problem.stridedBatched() && (inputs.a == nullptr || inputs.b == nullptr))
|
|
|| (!problem.stridedBatched()
|
|
&& (inputs.batchA == nullptr || inputs.batchB == nullptr))))
|
|
{
|
|
std::string matrixID = inputs.a == nullptr ? "A" : "B";
|
|
std::string msg = std::string("Unsupported nullptr for ") + matrixID
|
|
+ std::string(" when (Alpha !=0) && (K != 0)\n");
|
|
throw std::runtime_error(msg.c_str());
|
|
}
|
|
|
|
// Check if alpha matches problem definition
|
|
if(problem.alphaRestriction() != ScalarValue::Any
|
|
&& problem.alphaRestriction() != toScalarValueEnum(inputs.alpha))
|
|
{
|
|
std::stringstream inputValue;
|
|
inputValue << inputs.alpha;
|
|
std::string msg = std::string("Alpha value ") + inputValue.str()
|
|
+ std::string(" doesn't match that set in problem: ")
|
|
+ ToString(problem.alphaRestriction());
|
|
throw std::runtime_error(msg.c_str());
|
|
}
|
|
|
|
// Check if beta matches problem definition
|
|
if(problem.betaRestriction() != ScalarValue::Any
|
|
&& problem.betaRestriction() != toScalarValueEnum(inputs.beta))
|
|
{
|
|
std::stringstream inputValue;
|
|
inputValue << inputs.beta;
|
|
std::string msg = std::string("Beta value ") + inputValue.str()
|
|
+ std::string(" doesn't match that set in problem: ")
|
|
+ ToString(problem.betaRestriction());
|
|
throw std::runtime_error(msg.c_str());
|
|
}
|
|
|
|
if(problem.cEqualsD() && inputs.c != inputs.d)
|
|
throw std::runtime_error(
|
|
"ContractionProblem has cEqualsD set, but pointers for c and d are not equal");
|
|
|
|
std::vector<KernelInvocation> rv;
|
|
|
|
if(sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 0 && !debugSkipInit)
|
|
{
|
|
auto tiles = problem.getNumTiles(sizeMapping);
|
|
size_t skGrid = getSKGrid(problem, hardware, tiles);
|
|
// If problem can run full data-parallel, flags are not needed since there is no fixup step
|
|
// Skipping init kernel when flags are not needed saves significant time for small problems
|
|
if(tiles % skGrid != 0)
|
|
{
|
|
if(debug)
|
|
rv.push_back(
|
|
generateStreamKInitCall<TypedInputs, true>(problem, inputs, hardware));
|
|
else
|
|
rv.push_back(
|
|
generateStreamKInitCall<TypedInputs, false>(problem, inputs, hardware));
|
|
}
|
|
}
|
|
|
|
if((sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 1)
|
|
|| (sizeMapping.globalSplitU > 1 && sizeMapping.globalAccumulation != 2))
|
|
{
|
|
if(debug)
|
|
rv.push_back(generateBetaOnlyCall<TypedInputs, true>(problem, inputs, hardware));
|
|
else
|
|
rv.push_back(generateBetaOnlyCall<TypedInputs, false>(problem, inputs, hardware));
|
|
}
|
|
|
|
if(debug)
|
|
rv.push_back(generateSingleCall<TypedInputs, true>(problem, inputs, hardware));
|
|
else
|
|
rv.push_back(generateSingleCall<TypedInputs, false>(problem, inputs, hardware));
|
|
|
|
if(sizeMapping.globalAccumulation && !sizeMapping.streamK)
|
|
{
|
|
// TODO Streamk May need conversion call for HPA??
|
|
if(debug)
|
|
rv.push_back(
|
|
generateOutputConversionCall<TypedInputs, true>(problem, inputs, hardware));
|
|
else
|
|
rv.push_back(
|
|
generateOutputConversionCall<TypedInputs, false>(problem, inputs, hardware));
|
|
}
|
|
|
|
return rv;
|
|
}
|
|
|
|
bool ContractionSolution::getMatrixInstructionFromKernelName(
|
|
vector4<std::uint32_t>& matrixInst) const
|
|
{
|
|
|
|
std::string regexp_string("_MI(\\d+)x(\\d+)x(\\d+)x(\\d+)_");
|
|
std::regex miRegex(regexp_string);
|
|
std::smatch matches;
|
|
|
|
std::string kName = this->KernelName();
|
|
|
|
bool matched
|
|
= std::regex_search(kName, matches, miRegex)
|
|
&& (matches.size()
|
|
== 5 /* 1 (index 0) for entire match and 4 for the sub-experssion matches */);
|
|
|
|
if(matched)
|
|
{
|
|
// the first sub_match element (index 0) corresponds to the entire match
|
|
matrixInst.x = atoi(matches[1].str().c_str());
|
|
matrixInst.y = atoi(matches[2].str().c_str());
|
|
matrixInst.z = atoi(matches[3].str().c_str());
|
|
matrixInst.w = atoi(matches[4].str().c_str());
|
|
}
|
|
|
|
return matched;
|
|
}
|
|
|
|
bool ContractionSolution::getGSUAlgorithmFromKernelName(std::string& gsuAlg) const
|
|
{
|
|
std::string regexp_string("_GSUA([S|M])B_");
|
|
std::regex miRegex(regexp_string);
|
|
std::smatch matches;
|
|
|
|
std::string kName = this->KernelName();
|
|
|
|
bool matched
|
|
= std::regex_search(kName, matches, miRegex)
|
|
&& (matches.size()
|
|
== 2 /* 1 (index 0) for entire match and 1 for the sub-experssion matches */);
|
|
|
|
if(matched)
|
|
{
|
|
// the first sub_match element (index 0) corresponds to the entire match
|
|
gsuAlg = (matches[1].compare("S") == 0) ? "SingleBuffer" : "MultipleBuffer";
|
|
}
|
|
|
|
return matched;
|
|
}
|
|
|
|
std::vector<KernelInvocation>
|
|
ContractionSolution::solve(ContractionSolution::Problem const& problem,
|
|
ContractionSolution::Inputs const& inputs,
|
|
Hardware const& hardware) const
|
|
{
|
|
if(Debug::Instance().printWinningKernelName())
|
|
std::cout << "Running kernel: " << this->KernelName() << std::endl;
|
|
|
|
if(Debug::Instance().printKernelCommonParams())
|
|
{
|
|
std::cout << "Kernel name: " << this->KernelName() << std::endl;
|
|
std::cout << "Kernel parameters: " << std::endl;
|
|
|
|
vector4<std::uint32_t> matrixInst;
|
|
if(this->getMatrixInstructionFromKernelName(matrixInst))
|
|
std::cout << std::right << std::setw(30) << "MatrixInstruction: " << matrixInst
|
|
<< std::endl;
|
|
|
|
std::string GSUAlg;
|
|
if(this->getGSUAlgorithmFromKernelName(GSUAlg))
|
|
std::cout << std::right << std::setw(30) << "GSUAlgorithm: " << GSUAlg << std::endl;
|
|
|
|
std::cout << this->sizeMapping << std::endl;
|
|
}
|
|
// retreive alpha/beta type set via setAlpha/BetaType()
|
|
auto alphaType = problem.alphaType();
|
|
auto betaType = problem.betaType();
|
|
|
|
// TODO: Some gtests are passing the "problem" without actually defining the
|
|
// alpha/beta type (alphaType and betaType remain None).
|
|
// Until we fix those gtests, we need to keep this condition to adjust the missing
|
|
// alpha/beta data types.
|
|
if(alphaType == DataType::None)
|
|
{
|
|
alphaType
|
|
= problemType.aType == DataType::BFloat16 ? DataType::Float : problemType.dType;
|
|
}
|
|
if(betaType == DataType::None)
|
|
{
|
|
betaType = alphaType;
|
|
}
|
|
|
|
auto contractionInputsTypeId = ContractionInputs::TypeId(problemType.aType,
|
|
problemType.bType,
|
|
problemType.cType,
|
|
problemType.dType,
|
|
alphaType,
|
|
betaType);
|
|
|
|
switch(contractionInputsTypeId)
|
|
{
|
|
case ContractionInputs_S_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_S_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_D_D_D::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_D_D_D const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_C_C_C::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_C_C_C const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_Z_Z_Z::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_Z_Z_Z const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
#ifdef TENSILE_USE_HALF
|
|
case ContractionInputs_H_H_H::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_H_H_H const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_H_H_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_H_H_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_H_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_H_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
#endif // TENSILE_USE_HALF
|
|
case ContractionInputs_I8x4_I32_I32::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_I8x4_I32_I32 const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_I32_I32_I32::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_I32_I32_I32 const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_I8_I32_I32::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_I8_I32_I32 const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
#ifdef TENSILE_USE_BF16
|
|
case ContractionInputs_B_B_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B_B_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
#endif // TENSILE_USE_BF16
|
|
#ifdef TENSILE_USE_FP8_BF8
|
|
case ContractionInputs_F8_F8_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8_F8_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_F8_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8_B8_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8_B8_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
// hybrid cases: F8B8SS, B8F8SS
|
|
case ContractionInputs_F8B8_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8B8_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8F8_S_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8F8_S_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
// hybrid cases with To = B8
|
|
case ContractionInputs_F8B8_B8_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8B8_B8_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8F8_B8_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8F8_B8_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
// hybrid with F16 output
|
|
case ContractionInputs_F8_H_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8_H_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8_H_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8_H_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_F8B8_H_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_F8B8_H_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
case ContractionInputs_B8F8_H_S::TypeId():
|
|
{
|
|
auto const& typedInputs = dynamic_cast<ContractionInputs_B8F8_H_S const&>(inputs);
|
|
return solveTyped(problem, typedInputs, hardware);
|
|
}
|
|
#endif // TENSILE_USE_FP8_BF8
|
|
|
|
default:;
|
|
}
|
|
throw std::runtime_error("Data type not implemented.");
|
|
}
|
|
|
|
ContractionSolution::StaticPerformanceModel
|
|
ContractionSolution::staticPerformanceModel(double M,
|
|
double N,
|
|
double K,
|
|
double NumBatches,
|
|
double MT0,
|
|
double MT1,
|
|
double NumCUs,
|
|
double TotalGranularity,
|
|
int GlobalSplitU) const
|
|
{
|
|
StaticPerformanceModel spm;
|
|
|
|
int beta = (int)problemType.useBeta;
|
|
int betaReads = 0, betaWrites = 0;
|
|
if(GlobalSplitU == 1)
|
|
{
|
|
if(beta != 0.0)
|
|
betaReads = 1.0;
|
|
}
|
|
else
|
|
{
|
|
if(beta == 0)
|
|
betaWrites = 1; // zero output
|
|
else if(beta != 1.0) // if 1.0, just atomic update output
|
|
{
|
|
// if not 1.0, read, scale, write, then atomic update in kernel
|
|
betaReads = 1; // initial read for scale
|
|
betaWrites = 1; // writeback after scale
|
|
}
|
|
}
|
|
|
|
auto aInfo = DataTypeInfo::Get(problemType.aType);
|
|
auto bInfo = DataTypeInfo::Get(problemType.bType);
|
|
auto cInfo = DataTypeInfo::Get(problemType.cType);
|
|
auto dInfo = DataTypeInfo::Get(problemType.dType);
|
|
|
|
spm.memReadBytesA = (NumBatches * M * N * K) / MT1 * aInfo.elementSize;
|
|
spm.memReadBytesB = (NumBatches * M * N * K) / MT0 * bInfo.elementSize;
|
|
spm.memReadBytesC = (NumBatches * M * N) * betaReads * cInfo.elementSize;
|
|
|
|
if(GlobalSplitU == 1)
|
|
spm.memWriteBytesD = (NumBatches * M * N) * (1 + betaWrites) * dInfo.elementSize;
|
|
else
|
|
{
|
|
bool hardwareAtomic = false; // TODO-model
|
|
double atomicOperations = hardwareAtomic ? 2 : 3; // read-mod-write or cas //TODO-model
|
|
double atomicCollisions = 1.0; // TODO-could be based on K, GSU
|
|
spm.memWriteBytesD = (NumBatches * M * N)
|
|
* (betaWrites + atomicOperations * atomicCollisions)
|
|
* dInfo.elementSize;
|
|
}
|
|
spm.memReadBytes = spm.memReadBytesA + spm.memReadBytesB + spm.memReadBytesC;
|
|
spm.memGlobalReads = spm.memReadBytesA / aInfo.elementSize
|
|
+ spm.memReadBytesB / bInfo.elementSize
|
|
+ spm.memReadBytesC / cInfo.elementSize;
|
|
spm.memGlobalWrites = spm.memWriteBytesD / dInfo.elementSize;
|
|
|
|
return spm;
|
|
}
|
|
|
|
size_t ContractionSolution::requiredWorkspaceSize(Problem const& problem,
|
|
Hardware const& hardware) const
|
|
{
|
|
size_t size = 0;
|
|
|
|
if(sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 0)
|
|
{
|
|
auto tiles = problem.getNumTiles(sizeMapping);
|
|
size_t skGrid = getSKGrid(problem, hardware, tiles);
|
|
// Get space required for partial tiles
|
|
size += partialTileSize(skGrid);
|
|
// Add space for flags
|
|
// Flags for partial tiles - dword per flag for fast addressing and comparisons
|
|
// If tiles is evenly divided by grid size flags are not needed (DP mode)
|
|
if(tiles % skGrid != 0)
|
|
size += skGrid * 4;
|
|
// size *= batches; // TODO need tile and flag per batch
|
|
}
|
|
else
|
|
size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemC;
|
|
|
|
return size;
|
|
}
|
|
|
|
float lerp(float x, float x0, float x1, float y0, float y1)
|
|
{
|
|
return y0 + (x - x0) * (y1 - y0) / (x1 - x0);
|
|
}
|
|
|
|
int ContractionSolution::predictNumFullTiles(Problem const& problem,
|
|
Hardware const& hardware,
|
|
size_t tiles,
|
|
size_t skGrid) const
|
|
{
|
|
int itersPerTile = problem.getItersPerTile(sizeMapping);
|
|
int k = itersPerTile * sizeMapping.depthU;
|
|
// TODO Test if prediction should be normalized to iterations (DepthU) or full size
|
|
|
|
auto remainder = tiles % skGrid;
|
|
if(remainder == 0)
|
|
return 0;
|
|
|
|
// Temporary check for correct hardware until prediction curves are made for additional devices
|
|
AMDGPU const* pAMDGPU = dynamic_cast<AMDGPU const*>(&hardware);
|
|
assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0);
|
|
size_t cuCount = pAMDGPU->computeUnitCount;
|
|
if(cuCount != 304)
|
|
return 0;
|
|
|
|
// TODO Generalize code to allow prediction models for additional architectures
|
|
const std::vector<std::tuple<size_t, size_t>> predictionCurve = {{512, 304},
|
|
{768, 201},
|
|
{1024, 75},
|
|
{1536, 40},
|
|
{2048, 30},
|
|
{3072, 21},
|
|
{4096, 14},
|
|
{5120, 11},
|
|
{6144, 9},
|
|
{7168, 8},
|
|
{8192, 5},
|
|
{16384, 4}};
|
|
|
|
if(k < std::get<0>(predictionCurve[0]))
|
|
return 1;
|
|
|
|
for(int i = 1; i < predictionCurve.size(); ++i)
|
|
{
|
|
if(k < std::get<0>(predictionCurve[i]))
|
|
{
|
|
float threshold = lerp(k,
|
|
std::get<0>(predictionCurve[i - 1]),
|
|
std::get<0>(predictionCurve[i]),
|
|
std::get<1>(predictionCurve[i - 1]),
|
|
std::get<1>(predictionCurve[i]));
|
|
return remainder >= threshold ? 0 : 1;
|
|
}
|
|
}
|
|
|
|
return remainder >= std::get<1>(predictionCurve.back()) ? 0 : 1;
|
|
}
|
|
|
|
size_t ContractionSolution::getSKGrid(Problem const& problem,
|
|
Hardware const& hardware,
|
|
size_t tiles) const
|
|
{
|
|
AMDGPU const* pAMDGPU = dynamic_cast<AMDGPU const*>(&hardware);
|
|
|
|
assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0);
|
|
size_t cuCount = pAMDGPU->computeUnitCount;
|
|
|
|
// User-specified grid size for Stream-K kernel.
|
|
if(pAMDGPU->skFixedGrid > 0)
|
|
{
|
|
return pAMDGPU->skFixedGrid;
|
|
}
|
|
|
|
// Dynamically pick the minimum between the cuCount or number of tiles.
|
|
else if(pAMDGPU->skDynamicGrid == 1)
|
|
{
|
|
return min(cuCount, tiles);
|
|
}
|
|
|
|
// Dynamically pick the minimum between the cuCount or number of tiles,
|
|
// and scale down really large sizes to use fewer CUs for power/energy savings.
|
|
else if(pAMDGPU->skDynamicGrid == 2)
|
|
{
|
|
size_t skGrid = cuCount;
|
|
if(tiles > skGrid)
|
|
{
|
|
for(size_t i = 1; i <= 32; i *= 2)
|
|
{
|
|
size_t tilesPerCU = CeilDivide(i * tiles, cuCount);
|
|
size_t reducedGrid = CeilDivide(i * tiles, tilesPerCU);
|
|
float utilization = ((float)reducedGrid) / ((float)cuCount);
|
|
if(utilization > 0.75f)
|
|
{
|
|
if(utilization < 1.0f)
|
|
skGrid = reducedGrid;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return min(skGrid, tiles);
|
|
}
|
|
|
|
// Dynamically predict the best grid-size by weighing the cost of the fix-up
|
|
// step and the cost of processing MAC-loop instructions. When the cost of fix-up
|
|
// is the bottleneck, use smaller grid size.
|
|
// Architecture dependent.
|
|
else if(pAMDGPU->skDynamicGrid == 3)
|
|
{
|
|
size_t x = 1;
|
|
size_t y = 1;
|
|
size_t z = 1;
|
|
for(size_t i = 0; i < problem.freeIndicesA().size(); i++)
|
|
{
|
|
x *= problem.freeSizeA(i);
|
|
}
|
|
for(size_t i = 0; i < problem.freeIndicesB().size(); i++)
|
|
{
|
|
y *= problem.freeSizeB(i);
|
|
}
|
|
// TODO Batch dimension
|
|
for(size_t i = 0; i < problem.boundIndices().size(); ++i)
|
|
{
|
|
z *= problem.boundSize(i);
|
|
}
|
|
|
|
return streamk::best_predicted_grid_size(sizeMapping.macroTile.x,
|
|
sizeMapping.macroTile.y,
|
|
sizeMapping.depthU,
|
|
x,
|
|
y,
|
|
z,
|
|
1,
|
|
cuCount);
|
|
}
|
|
|
|
// Limit the CUs Stream-K is launched on either max or the specified,
|
|
// whichever is minimum.
|
|
else if(pAMDGPU->skMaxCUs > 0)
|
|
{
|
|
return min(cuCount, static_cast<size_t>(pAMDGPU->skMaxCUs));
|
|
}
|
|
|
|
// Multiply the cuCount with a constant factor (c), and launch
|
|
// c * cuCount number of workgroups for Stream-K.
|
|
else if(pAMDGPU->skGridMultiplier > 1)
|
|
{
|
|
return cuCount * pAMDGPU->skGridMultiplier;
|
|
}
|
|
|
|
// If no option is specified, launch exactly cuCount worth of workgroups.
|
|
else
|
|
{
|
|
return cuCount;
|
|
}
|
|
}
|
|
|
|
size_t ContractionSolution::partialTileSize(size_t skGrid) const
|
|
{
|
|
size_t size = 0;
|
|
|
|
size_t tileSize
|
|
= sizeMapping.macroTile.x * sizeMapping.macroTile.y * sizeMapping.workspaceSizePerElemC;
|
|
size += tileSize * skGrid; // Partials tile per WG
|
|
// TODO batches
|
|
// TODO round up for alignment?
|
|
|
|
return size;
|
|
}
|
|
|
|
float ContractionSolution::computeGranularity(float x)
|
|
{
|
|
return x / ceil(x);
|
|
}
|
|
|
|
ContractionSolution::Granularities ContractionSolution::computeGranularities(
|
|
Hardware const& hardware, double M, double N, double K, double NumBatches) const
|
|
{
|
|
ContractionSolution::Granularities granularities;
|
|
|
|
double MT0 = sizeMapping.macroTile.x;
|
|
double MT1 = sizeMapping.macroTile.y;
|
|
|
|
AMDGPU const* pAMDGPU = dynamic_cast<AMDGPU const*>(&hardware);
|
|
assert(pAMDGPU);
|
|
|
|
double NumCUs = pAMDGPU->computeUnitCount;
|
|
double wavefrontSize = pAMDGPU->wavefrontSize;
|
|
double simdPerCu = pAMDGPU->simdPerCu;
|
|
|
|
double GlobalSplitU = sizeMapping.globalSplitU;
|
|
double LocalSplitU = sizeMapping.workGroupSize.z;
|
|
|
|
granularities.MT0 = MT0;
|
|
granularities.MT1 = MT1;
|
|
granularities.GSU = GlobalSplitU;
|
|
granularities.LSU = LocalSplitU;
|
|
granularities.CUs = NumCUs;
|
|
|
|
granularities.numTiles0 = M / MT0;
|
|
granularities.numTiles1 = N / MT1;
|
|
|
|
granularities.tile0Granularity = computeGranularity(granularities.numTiles0);
|
|
granularities.tile1Granularity = computeGranularity(granularities.numTiles1);
|
|
|
|
granularities.tilesPerCu
|
|
= (NumBatches * ceil(granularities.numTiles0) * ceil(granularities.numTiles1))
|
|
/ (NumCUs / GlobalSplitU / LocalSplitU);
|
|
|
|
granularities.totalTiles = ceil(granularities.numTiles0) * ceil(granularities.numTiles1);
|
|
granularities.natTilesPerCu = NumBatches * granularities.totalTiles / NumCUs;
|
|
granularities.suTilesPerCu = (granularities.totalTiles * GlobalSplitU) / NumCUs;
|
|
granularities.suCuGranularity = computeGranularity(granularities.suTilesPerCu);
|
|
|
|
granularities.waveGranularity = std::min(
|
|
1.00,
|
|
static_cast<double>(floor(granularities.tilesPerCu + 1.0) * sizeMapping.workGroupSize.x
|
|
* sizeMapping.workGroupSize.y * sizeMapping.workGroupSize.z)
|
|
/ pAMDGPU->wavefrontSize / pAMDGPU->simdPerCu);
|
|
|
|
granularities.waves
|
|
= ceil((sizeMapping.workGroupSize.x * sizeMapping.workGroupSize.y) / wavefrontSize);
|
|
|
|
granularities.suWavesPerSimdx2
|
|
= (granularities.suTilesPerCu * granularities.waves) / (2 * simdPerCu);
|
|
granularities.suWaveGranularity
|
|
= granularities.suWavesPerSimdx2 * ceil(granularities.suWavesPerSimdx2);
|
|
|
|
double nat_tiles_per_cu
|
|
= NumBatches * ceil(granularities.numTiles0) * ceil(granularities.numTiles1) / NumCUs;
|
|
granularities.natCuGranularity = ceil(nat_tiles_per_cu) * ceil(nat_tiles_per_cu) / NumCUs;
|
|
|
|
granularities.cuGranularity = computeGranularity(granularities.tilesPerCu);
|
|
|
|
granularities.totalGranularity
|
|
= granularities.tile0Granularity * granularities.tile1Granularity
|
|
* granularities.cuGranularity * granularities.waveGranularity;
|
|
|
|
granularities.totalTileAwareGranularity
|
|
= granularities.tile0Granularity * granularities.tile1Granularity
|
|
* granularities.suCuGranularity * granularities.suWaveGranularity;
|
|
|
|
return granularities;
|
|
}
|
|
|
|
ContractionSolution::ProjectedPerformance
|
|
ContractionSolution::projectedPerformance(Problem const& problem,
|
|
Hardware const& hardware) const
|
|
{
|
|
ProjectedPerformance pp;
|
|
|
|
double M = 1.0, N = 1.0;
|
|
if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesA(problem, sizeMapping.packBatchDims);
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++)
|
|
M *= problem.a().sizes()[*pi];
|
|
}
|
|
else
|
|
M = problem.freeSizeA(0);
|
|
|
|
if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesB(problem, sizeMapping.packBatchDims);
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++)
|
|
N *= problem.b().sizes()[*pi];
|
|
}
|
|
else
|
|
N = problem.freeSizeB(0);
|
|
|
|
double NumBatches = 1;
|
|
if(sizeMapping.packBatchDims == 0)
|
|
{
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
NumBatches *= problem.batchSize(i);
|
|
}
|
|
double K = problem.boundSize(0); // TODO - fix for multiple summations
|
|
|
|
pp.granularities = ContractionSolution::computeGranularities(hardware, M, N, K, NumBatches);
|
|
|
|
auto it = ideals.begin();
|
|
|
|
int closestKMeasure = std::numeric_limits<int>::max();
|
|
double closestKPerformance = 0.0;
|
|
|
|
while(it != ideals.end())
|
|
{
|
|
int myK = it->first;
|
|
int myMeasure = std::abs(myK - K);
|
|
if(myMeasure < closestKMeasure)
|
|
{
|
|
closestKMeasure = myMeasure;
|
|
closestKPerformance = it->second;
|
|
}
|
|
it++;
|
|
}
|
|
|
|
double MT0 = pp.granularities.MT0;
|
|
double MT1 = pp.granularities.MT1;
|
|
double NumCUs = pp.granularities.CUs;
|
|
|
|
double GlobalSplitU = pp.granularities.GSU;
|
|
double IdealGranularityPerf = closestKPerformance;
|
|
|
|
pp.staticModel = staticPerformanceModel(
|
|
M, N, K, NumBatches, MT0, MT1, NumCUs, pp.granularities.totalGranularity, GlobalSplitU);
|
|
|
|
pp.speedGFlops = IdealGranularityPerf * pp.granularities.totalGranularity;
|
|
pp.CUs = NumCUs;
|
|
|
|
return pp;
|
|
}
|
|
|
|
ContractionSolution::TAMetricProblemScore ContractionSolution::computeProblemScore(
|
|
Hardware const& hardware, double M, double N, double K, double NumBatches) const
|
|
{
|
|
ContractionSolution::TAMetricProblemScore pp;
|
|
pp.granularites = ContractionSolution::computeGranularities(hardware, M, N, K, NumBatches);
|
|
|
|
pp.M = M;
|
|
pp.N = N;
|
|
pp.K = K;
|
|
|
|
double slope = linearModel.slope;
|
|
double intercept = linearModel.intercept;
|
|
double perf_max = linearModel.max;
|
|
|
|
double sum_value = K;
|
|
double sum_perf0 = sum_value / (intercept + (slope * sum_value));
|
|
pp.summationPerformance = 1000.0 * sum_perf0 / perf_max;
|
|
|
|
return pp;
|
|
}
|
|
|
|
double ContractionSolution::computeTileAwareMetric(
|
|
ContractionSolution::TAMetricProblemScore pp,
|
|
ContractionSolution::TAMetricProblemScore ppReference) const
|
|
{
|
|
double tile0GranularityDim = abs(log(ppReference.granularites.tile0Granularity)
|
|
- log(pp.granularites.tile0Granularity));
|
|
double metric = tile0GranularityDim;
|
|
|
|
double tile1GranularityDim = abs(log(ppReference.granularites.tile1Granularity)
|
|
- log(pp.granularites.tile1Granularity));
|
|
metric += tile1GranularityDim;
|
|
|
|
double natCuGranularityDim = abs(log(ppReference.granularites.natCuGranularity)
|
|
- log(pp.granularites.natCuGranularity));
|
|
metric += natCuGranularityDim;
|
|
|
|
double suCuGranularityDim = abs(log(ppReference.granularites.suCuGranularity)
|
|
- log(pp.granularites.suCuGranularity));
|
|
metric += suCuGranularityDim;
|
|
|
|
double suWaveGranularityDim = abs(log(ppReference.granularites.suWaveGranularity)
|
|
- log(pp.granularites.suWaveGranularity));
|
|
metric += suWaveGranularityDim;
|
|
|
|
double natTilesPerCuDim
|
|
= abs(log(ppReference.granularites.natTilesPerCu) - log(pp.granularites.natTilesPerCu));
|
|
metric += natTilesPerCuDim;
|
|
|
|
double suTilesPerCuDim
|
|
= abs(log(ppReference.granularites.suTilesPerCu) - log(pp.granularites.suTilesPerCu));
|
|
metric += suTilesPerCuDim;
|
|
|
|
double summationPerformanceDim
|
|
= abs(ppReference.summationPerformance - pp.summationPerformance);
|
|
metric += summationPerformanceDim;
|
|
|
|
return metric;
|
|
}
|
|
|
|
double ContractionSolution::computeTAMScore(Problem const& problem,
|
|
Hardware const& hardware,
|
|
double model_M,
|
|
double model_N,
|
|
double model_K,
|
|
double model_NumBatches) const
|
|
{
|
|
double M = 1.0, N = 1.0;
|
|
if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesA(problem, sizeMapping.packBatchDims);
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++)
|
|
M *= problem.a().sizes()[*pi];
|
|
}
|
|
else
|
|
M = problem.freeSizeA(0);
|
|
|
|
if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2)
|
|
{
|
|
std::vector<size_t> packedIndices
|
|
= generatePackedIndicesB(problem, sizeMapping.packBatchDims);
|
|
for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++)
|
|
N *= problem.b().sizes()[*pi];
|
|
}
|
|
else
|
|
N = problem.freeSizeB(0);
|
|
|
|
double NumBatches = 1;
|
|
if(sizeMapping.packBatchDims == 0)
|
|
{
|
|
for(size_t i = 0; i < problem.batchIndices().size(); i++)
|
|
NumBatches *= problem.batchSize(i);
|
|
}
|
|
double K = problem.boundSize(0); // TODO - fix for multiple summations
|
|
|
|
ContractionSolution::TAMetricProblemScore pp
|
|
= computeProblemScore(hardware, M, N, K, NumBatches);
|
|
|
|
ContractionSolution::TAMetricProblemScore ppReference
|
|
= computeProblemScore(hardware, model_M, model_N, model_K, model_NumBatches);
|
|
|
|
double distance = computeTileAwareMetric(pp, ppReference);
|
|
|
|
return distance;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& stream,
|
|
ContractionSolution::StaticPerformanceModel const& spm)
|
|
{
|
|
return stream << " memReadBytesA=" << spm.memReadBytesA
|
|
<< " memReadBytesB=" << spm.memReadBytesB
|
|
<< " memReadBytesC=" << spm.memReadBytesC
|
|
<< " memWriteBytesD=" << spm.memWriteBytesD;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& stream,
|
|
ContractionSolution::ProjectedPerformance const& pp)
|
|
{
|
|
return stream << " numTiles0=" << pp.granularities.numTiles0
|
|
<< " numTiles1=" << pp.granularities.numTiles1
|
|
<< " tilesPerCu=" << pp.granularities.tilesPerCu
|
|
|
|
<< " totalGranularity=" << pp.granularities.totalGranularity
|
|
<< " tile0Granularity=" << pp.granularities.tile0Granularity
|
|
<< " tile1Granularity=" << pp.granularities.tile1Granularity
|
|
<< " cuGranularity=" << pp.granularities.cuGranularity
|
|
<< " waveGranularity=" << pp.granularities.waveGranularity
|
|
|
|
<< " speedGFlops=" << pp.speedGFlops
|
|
|
|
<< " staticModel=[ " << pp.staticModel << " ]";
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& stream, BufferLoadCheckPacket const& st)
|
|
{
|
|
return stream << " shiftPtrElemA=" << st.shiftPtrElemA
|
|
<< " shiftPtrElemB=" << st.shiftPtrElemB << " depthUorMT0=" << st.depthUorMT0
|
|
<< " depthUorMT1=" << st.depthUorMT1;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& stream, const SizeMapping& sizeMapping)
|
|
{
|
|
std::ios_base::fmtflags flags(stream.flags());
|
|
|
|
stream << std::right << std::setw(30) << "workGroupSize: " << sizeMapping.workGroupSize
|
|
<< std::endl
|
|
<< std::setw(30) << "threadTile: " << sizeMapping.threadTile << std::endl
|
|
<< std::setw(30) << "macroTile: " << sizeMapping.macroTile << std::endl
|
|
|
|
<< std::setw(30) << "staggerU: " << sizeMapping.staggerU << std::endl
|
|
<< std::setw(30) << "depthU: " << sizeMapping.depthU << std::endl
|
|
<< std::setw(30) << "globalSplitU: " << sizeMapping.globalSplitU << std::endl
|
|
<< std::setw(30) << "staggerStrideShift: " << sizeMapping.staggerStrideShift
|
|
<< std::endl
|
|
<< std::setw(30) << "workGroupMapping: " << sizeMapping.workGroupMapping << std::endl
|
|
|
|
<< std::setw(30) << "packBatchDims: " << sizeMapping.packBatchDims << std::endl
|
|
<< std::setw(30) << "packSummationDims: " << sizeMapping.packSummationDims
|
|
<< std::endl
|
|
<< std::setw(30) << "magicDivAlg: " << sizeMapping.magicDivAlg << std::endl
|
|
<< std::setw(30) << "streamK: " << sizeMapping.streamK << std::endl
|
|
<< std::setw(30) << "streamKAtomic: " << sizeMapping.streamKAtomic << std::endl
|
|
<< std::setw(30) << "persistentKernel: " << sizeMapping.persistentKernel << std::endl
|
|
<< std::setw(30)
|
|
<< "persistentKernelAlongBatch: " << sizeMapping.persistentKernelAlongBatch
|
|
<< std::endl
|
|
|
|
<< std::setw(30) << "sourceKernel: " << sizeMapping.sourceKernel << std::endl
|
|
<< std::setw(30) << "globalAccumulation: " << sizeMapping.globalAccumulation
|
|
<< std::endl
|
|
<< std::setw(30) << "workspaceSizePerElemC: " << sizeMapping.workspaceSizePerElemC
|
|
<< std::endl;
|
|
|
|
stream.flags(flags);
|
|
|
|
return stream;
|
|
}
|
|
} // namespace Tensile
|