// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include namespace rocRoller { namespace KernelGraph { namespace AddTDMToLDSDetail { std::vector> findCandidates(KernelGraph const& kgraph, ContextPtr context) { using namespace ControlGraph; using namespace CoordinateGraph; auto isTDMToLDSLoadTiled = [&kgraph](int tag) { bool rv = false; if(kgraph.control.get(tag)) { auto macroTile = kgraph.coordinates.get(kgraph.mapper.get(tag)); rv = macroTile && macroTile->memoryType == MemoryType::WAVE_TDMToLDS; } return rv; }; return getLoadTiledStoreLDSTilePairs(kgraph, isTDMToLDSLoadTiled); } void replaceLoadTiled(KernelGraph& kgraph, int loadTiledTag, int storeLDSTileTag, int loadTiledTDMToLDSTag) { using namespace ControlGraph; auto codegen = getCodeGeneratorCoordinates(kgraph, storeLDSTileTag); moveConnections(kgraph, loadTiledTag, loadTiledTDMToLDSTag, 0); moveConnections(kgraph, storeLDSTileTag, loadTiledTDMToLDSTag, codegen.size()); replaceWith(kgraph, loadTiledTag, loadTiledTDMToLDSTag, false); Log::debug(" AddTDMToLDS: Replace LoadTiled({}) with LoadTiledTDMToLDS({}).", loadTiledTag, loadTiledTDMToLDSTag); } } /** @brief This transformation replaces LoadTiled that load MacroTiles * of WAVE_TDMToLDS with LoadTiledTDMToLDS and turns the respective * StoreLDSTile operations store said MacroTiles in LDS into NOPs. */ KernelGraph AddTDMToLDS::apply(KernelGraph const& original) { using namespace ControlGraph; using namespace AddTDMToLDSDetail; Log::debug("KernelGraph::Transforms::AddTDMToLDS::apply()"); const auto& arch = m_context->targetArchitecture(); const auto hasTDM = arch.HasCapability(GPUCapability::HasTDM); if(!hasTDM) { Log::debug( fmt::format(" AddTDMToLDS: Target {} does not support TDM. Nothing to do.\n", toString(arch.target()))); return original; } const auto candidates = findCandidates(original, m_context); if(std::ranges::empty(candidates)) { Log::debug(" AddTDMToLDS: No candidates found. Nothing to do.\n"); return original; } auto kgraph{original}; std::unordered_set nodesToPurge; for(auto [loadTiledTag, storeLDSTileTag] : candidates) { Log::debug(fmt::format(" AddTDMToLDS: found LoadTiled({}) and StoreLDSTile({})\n", loadTiledTag, storeLDSTileTag)); auto variableType = getVariableType(kgraph, loadTiledTag); auto loadTiledTDMToLDSTag = kgraph.control.addElement(LoadTiledTDMToLDS(variableType)); replaceLoadTiled(kgraph, loadTiledTag, storeLDSTileTag, loadTiledTDMToLDSTag); nodesToPurge.insert(loadTiledTag); if(nodesToPurge.count(storeLDSTileTag) == 0) { replaceWith(kgraph, storeLDSTileTag, kgraph.control.addElement(NOP()), false); Log::debug(" AddTDMToLDS: Replace StoreLDSTile({}) with NOP.", storeLDSTileTag); nodesToPurge.insert(storeLDSTileTag); } } for(auto node : nodesToPurge) { purgeNodes(kgraph, {node}); } return kgraph; } ConstraintStatus NoTDMToLDSTile(const KernelGraph& kgraph) { TIMER(t, "Constraint::NoTDMToLDSTile"); using namespace ControlGraph; using namespace CoordinateGraph; ConstraintStatus retval; auto isLoadTiledOrStoreLDSTileOfTDMToLDSTile = [&kgraph](int tag) { bool rv = false; if(kgraph.control.get(tag)) { auto macroTile = kgraph.coordinates.get(kgraph.mapper.get(tag)); rv = macroTile && macroTile->memoryType == MemoryType::WAVE_TDMToLDS; } else if(kgraph.control.get(tag)) { auto macroTile = kgraph.coordinates.get(kgraph.mapper.get(tag)); rv = macroTile && macroTile->memoryType == MemoryType::WAVE_TDMToLDS; } return rv; }; // Post-check: ensure all LDS LoadTiled and StoreLDSTiled have been replaced const auto unexpectedOps = filter(isLoadTiledOrStoreLDSTileOfTDMToLDSTile, kgraph.control.getNodes()) .to(); retval.combine(std::ranges::empty(unexpectedOps), concatenate("Found LoadTiled or StoreLDSTiled of WAVE_TDMToLDS Tile: ", unexpectedOps)); return retval; } std::vector AddTDMToLDS::postConstraints() const { return {NoTDMToLDSTile}; } } }