// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include #include namespace rocRoller { namespace KernelGraph { using namespace CoordinateGraph; using GD = rocRoller::Graph::Direction; namespace RemapOutputTilesDetail { void TileSizeInfo::recordSize(int dim, int tileNumTag, auto direction, auto expr) { // TODO: Is there a way to make this safer? // // The result of, for example: // // matrixSize(M) / tileSize(M) // // should fit within a Int32 AssertFatal(0 <= dim && dim < 3, ShowValue(dim)); if(expr != nullptr) { if(sizes[dim] == nullptr) { sizes[dim] = convert(DataType::Int32, expr); } else { // They aren't alway identical, but they // should be equivalent after resolving // command arguments. // // For example: A_size_1 is K; and so is B_size_0 // // TODO: Emit a command predicate to enforce this } } danglers[{dim, direction}].insert(tileNumTag); } TileSizeInfo getTileSizeInfo(KernelGraph const& kgraph) { TileSizeInfo info; auto tileNumTags = kgraph.coordinates.getNodes().to(); for(auto const& tileNumTag : tileNumTags) { if(std::empty(kgraph.coordinates.getNeighbours(tileNumTag))) { // If we have no downstream neighbours, we // will create a new workgroup below this, and // look upstream auto tileNum = *kgraph.coordinates.get(tileNumTag); info.recordSize(tileNum.dim, tileNumTag, GD::Upstream, tileNum.size); } if(std::empty(kgraph.coordinates.getNeighbours(tileNumTag))) { auto tileNum = *kgraph.coordinates.get(tileNumTag); info.recordSize(tileNum.dim, tileNumTag, GD::Downstream, tileNum.size); } } return info; } int workgroupDimensions(TileSizeInfo const& info) { if(info.sizes[0] != nullptr && info.sizes[1] != nullptr && info.sizes[2] != nullptr) return 3; if(info.sizes[0] != nullptr && info.sizes[1] != nullptr) return 2; if(info.sizes[0] != nullptr) return 1; Throw("Invalid number of dimensions."); } Expression::ExpressionPtr totalNumberOfWorkgroups(TileSizeInfo const& info) { AssertFatal(info.sizes[0] != nullptr); auto rv = info.sizes[0]; for(int i = 1; i < 3; ++i) if(info.sizes[i] != nullptr) rv = rv * info.sizes[i]; return rv; } void connectWorkgroupsWithMapping(TileSizeInfo const& info, rocRoller::KernelGraph::KernelGraph& graph, int dimension, Expression::ExpressionPtr size) { auto totalSize = totalNumberOfWorkgroups(info); auto numDims = workgroupDimensions(info); AssertFatal(numDims == 2, ShowValue(numDims)); AssertFatal(dimension == 0 || dimension == 1, ShowValue(dimension)); for(auto direction : {GD::Downstream, GD::Upstream}) { // Downstream: Starting at wgTop looking down // Upstream: Starting at wgBot looking up auto remappedDims = workgroupMapping(info, graph, direction, dimension, size); std::array tileNumTags = {remappedDims.parallelDim, remappedDims.perpendicularDim}; for(auto dim = 0; dim < numDims; ++dim) { for(auto tileNumTag : info.danglers.at({dim, direction})) { if(direction == GD::Upstream) { Log::debug("KernelGraph::RemapOutputTiles: Adding PassThrough " "from tile {} to mapped-tile {} (size {})", tileNumTag, tileNumTags[dim], toString(info.sizes[dim])); graph.coordinates.addElement( PassThrough(), {tileNumTag}, {tileNumTags[dim]}); } else { Log::debug("KernelGraph::RemapOutputTiles: Adding PassThrough " "from mapped-tile {} (size {}) to tile {}", tileNumTags[dim], toString(info.sizes[dim]), tileNumTag); graph.coordinates.addElement( PassThrough(), {tileNumTags[dim]}, {tileNumTag}); } } } } } RemappedDimensions workgroupMapping(TileSizeInfo const& info, rocRoller::KernelGraph::KernelGraph& graph, GD direction, uint dimension, Expression::ExpressionPtr size) { AssertFatal(dimension == 0 || dimension == 1); AssertFatal(workgroupDimensions(info) == 2); auto totalSize = totalNumberOfWorkgroups(info); // This MacroTileNumber will be connected to a workgroup in // later transformation or be used in AddStreamK with dimension // K to create a flattened tile space. auto workgroup = graph.coordinates.addElement(MacroTileNumber(0, totalSize, nullptr)); // Downstream: Starting at workgroup looking down (forward transform) // Upstream: Starting at workgroup looking up (reverse transform) using ExpressionPtr = Expression::ExpressionPtr; using ExpressionPtrPair = std::pair; using ExpressionPtrVectorPair = std::pair, std::vector>; auto one = Expression::literal(1); auto zero = Expression::literal(0); auto parallelSize = info.sizes[dimension]; auto perpendicularSize = info.sizes[1 - dimension]; auto blockSize = convert(DataType::Int32, size * perpendicularSize); setComment(blockSize, "WGM block size"); auto mainBlockSize = convert(DataType::Int32, totalSize / blockSize) * blockSize; setComment(mainBlockSize, "WGM main block size"); auto tailBlockSize = convert(DataType::Int32, parallelSize % size); setComment(tailBlockSize, "WGM tail block size"); auto groupNumber = graph.coordinates.addElement(Linear()); auto groupIndex = graph.coordinates.addElement(Linear(size, nullptr)); auto blockNumber = graph.coordinates.addElement(Linear()); auto blockIndex = graph.coordinates.addElement(Linear(perpendicularSize, nullptr)); auto mainBlockNumber = graph.coordinates.addElement(Linear()); auto mainBlockIndex = graph.coordinates.addElement(Linear(mainBlockSize, nullptr)); auto tailBlockNumber = graph.coordinates.addElement(Linear()); auto tailBlockIndex = graph.coordinates.addElement(Linear(tailBlockSize, nullptr)); auto parallel = graph.coordinates.addElement(Linear(parallelSize, nullptr)); auto perpendicular = graph.coordinates.addElement(Linear(perpendicularSize, nullptr)); // 0 argument is mainBlockNumber auto condition = Expression::positionalArgument(0, Register::Type::Scalar, DataType::UInt32) == Expression::literal(0); ExpressionPtrVectorPair stridesParallel{{zero, size, one, zero}, {zero, zero, zero, one}}; ExpressionPtrPair initialValuesParallel{ nullptr, convert(DataType::Int32, parallelSize / size) * size}; ExpressionPtrVectorPair stridesPerpendicular{{zero, one, zero}, {zero, zero, one}}; ExpressionPtrPair initialValuesPerpendicular{nullptr, nullptr}; if(direction == GD::Upstream) { graph.coordinates.addElement(PiecewiseAffineJoin(condition, stridesPerpendicular, initialValuesPerpendicular), {perpendicular}, {mainBlockNumber, blockIndex, tailBlockNumber}); graph.coordinates.addElement( PiecewiseAffineJoin(condition, stridesParallel, initialValuesParallel), {parallel}, {mainBlockNumber, blockNumber, groupIndex, tailBlockIndex}); graph.coordinates.addElement( Flatten(), {tailBlockNumber, tailBlockIndex}, {mainBlockIndex}); graph.coordinates.addElement( Flatten(), {blockNumber, blockIndex}, {groupNumber}); graph.coordinates.addElement(Flatten(), {groupNumber, groupIndex}, {workgroup}); graph.coordinates.addElement( Flatten(), {mainBlockNumber, mainBlockIndex}, {workgroup}); } else { graph.coordinates.addElement( Tile(), {workgroup}, {mainBlockNumber, mainBlockIndex}); graph.coordinates.addElement(Tile(), {workgroup}, {groupNumber, groupIndex}); graph.coordinates.addElement(Tile(), {groupNumber}, {blockNumber, blockIndex}); graph.coordinates.addElement( Tile(), {mainBlockIndex}, {tailBlockNumber, tailBlockIndex}); graph.coordinates.addElement( PiecewiseAffineJoin(condition, stridesParallel, initialValuesParallel), {mainBlockNumber, blockNumber, groupIndex, tailBlockIndex}, {parallel}); graph.coordinates.addElement(PiecewiseAffineJoin(condition, stridesPerpendicular, initialValuesPerpendicular), {mainBlockNumber, blockIndex, tailBlockNumber}, {perpendicular}); } if(dimension == 0) return {workgroup, parallel, perpendicular}; return {workgroup, perpendicular, parallel}; } } RemapOutputTiles::RemapOutputTiles(std::optional workgroupMappingDim, Expression::ExpressionPtr workgroupMappingValue) : m_workgroupMappingDim(workgroupMappingDim) , m_workgroupMappingValue(workgroupMappingValue) { } KernelGraph RemapOutputTiles::apply(KernelGraph const& original) { using namespace RemapOutputTilesDetail; auto kgraph = original; if(m_workgroupMappingDim.has_value()) { auto info = getTileSizeInfo(original); connectWorkgroupsWithMapping( info, kgraph, m_workgroupMappingDim.value(), m_workgroupMappingValue); } return kgraph; } } }