// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include #include #include #include namespace rocRoller { namespace KernelGraph { using namespace ControlGraph; using namespace CoordinateGraph; using GD = rocRoller::Graph::Direction; std::string LoopOverTileNumbers::name() const { return concatenate("LoopOverTileNumbers"); } LoopOverTileNumbers::LoopOverTileNumbers(std::vector const& dims, std::vector const& tileNumberCoordSizes, uint numIteratedTiles, std::string const& topLoop, ContextPtr context) : m_dimensions(dims) , m_topLoop(topLoop) , m_context(context) { for(auto s : tileNumberCoordSizes) m_tileNumberCoordSizes.push_back(Expression::literal(s)); m_numIteratedTileNumbers = Expression::literal(numIteratedTiles); } // // Stage // // Look for all leaf MacroTileNumbers with matching // sub-dimension. // // Matches are: tile->dim in m_dimensions. // void LoopOverTileNumbers::stage(KernelGraph const& graph) { // Find all dangling MacroTileNumber dimensions associated // with the requested dimensions for(auto dimension : m_dimensions) { auto danglingTileNumberPredicate = [&](int tag) { auto maybeTileNumber = graph.coordinates.get(tag); if(!maybeTileNumber) return false; if(maybeTileNumber->dim != dimension) return false; if(std::empty(graph.coordinates.getNeighbours(tag)) || std::empty(graph.coordinates.getNeighbours(tag))) return true; return false; }; m_tileNumberCoords[dimension] = graph.coordinates.findElements(danglingTileNumberPredicate) .to(); } } // // Commit // void LoopOverTileNumbers::commit(KernelGraph& graph) { auto numTotalTiles = Expression::literal(1u); for(auto d : m_dimensions) numTotalTiles = numTotalTiles * m_tileNumberCoordSizes[d]; numTotalTiles = simplify(numTotalTiles); auto [forLoopCoord, forLoopOp] = rangeFor(graph, m_numIteratedTileNumbers, "TileLoop", DataType::UInt32); // Create forward/reverse tile-numbers for each dimension // and attach to all staged tile-number coordinates std::vector tileNumbersUp, tileNumbersDown; for(auto d : m_dimensions) { auto up = graph.coordinates.addElement( MacroTileNumber(d, m_tileNumberCoordSizes[d], nullptr)); auto down = graph.coordinates.addElement( MacroTileNumber(d, m_tileNumberCoordSizes[d], nullptr)); for(auto tileNumTag : m_tileNumberCoords.at(d)) { if(std::empty(graph.coordinates.getNeighbours(tileNumTag))) graph.coordinates.addElement(PassThrough(), {down}, {tileNumTag}); if(std::empty(graph.coordinates.getNeighbours(tileNumTag))) graph.coordinates.addElement(PassThrough(), {tileNumTag}, {up}); } tileNumbersUp.push_back(up); tileNumbersDown.push_back(down); } // Create foward/reverse transforms from (Workgroup, // ForLoop) to new tile-numbers auto workgroups = simplify(numTotalTiles / m_numIteratedTileNumbers); auto initUp = graph.coordinates.addElement(MacroTileNumber(0, workgroups, nullptr)); auto flatUp = graph.coordinates.addElement(Linear(numTotalTiles, nullptr)); graph.coordinates.addElement(Flatten(), {forLoopCoord, initUp}, {flatUp}); graph.coordinates.addElement(Tile(), std::vector{flatUp}, tileNumbersDown); auto initDown = graph.coordinates.addElement(MacroTileNumber(0, workgroups, nullptr)); auto flatDown = graph.coordinates.addElement(Linear(numTotalTiles, nullptr)); graph.coordinates.addElement(Tile(), {flatDown}, {forLoopCoord, initDown}); graph.coordinates.addElement(Flatten(), tileNumbersUp, std::vector{flatDown}); // Insert ForLoop into graph insertWithBody(graph, m_topLoopOp, forLoopOp); } // // Apply // KernelGraph LoopOverTileNumbers::apply(KernelGraph const& original) { TIMER(t, "KernelGraph::LoopOverTileNumbers"); // Make sure we can find the top-for-loop location auto findTopLoopPredicate = [&](int tag) -> bool { auto maybeForLoop = original.control.get(tag); if(!maybeForLoop) return false; if(maybeForLoop->loopName == m_topLoop) return true; return false; }; auto maybeTopLoopOp = original.control.findNodes( *original.control.roots().begin(), findTopLoopPredicate, GD::Downstream); if(maybeTopLoopOp.empty()) { rocRoller::Log::getLogger()->warn( "Unable to find ForLoop '{}' during LoopOverTileNumbers pass. " "LoopOverTileNumbers transform skipped.", m_topLoop); return original; } m_topLoopOp = *maybeTopLoopOp.take(1).only(); auto graph = original; stage(graph); commit(graph); return graph; } } }