// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include #include #include #include namespace rocRoller::KernelGraph { using namespace ControlGraph; /** * @brief Look for {Assign Multiply(., .)} --Sequence--> {Assign Add(., .)}. * * Look for * * Assign Multiply(., .) -- Sequence --> Assign Add(., .) * * Make sure only one DF edge out of the result of the multiply. */ std::optional> findMultiplyAdd(KernelGraph const& kgraph, std::unordered_set const& exclude) { namespace CT = rocRoller::KernelGraph::CoordinateGraph; for(auto const& parent : kgraph.control.getNodes()) { if(exclude.contains(parent)) continue; // find Multiply auto multiplyAssign = kgraph.control.get(parent); if(!std::holds_alternative(*multiplyAssign->expression)) continue; auto child = only(kgraph.control.getOutputNodeIndices(parent)); if(!child) continue; auto addAssign = kgraph.control.get(*child); if(!addAssign) continue; auto dst = kgraph.mapper.get(parent, NaryArgument::DEST); AssertFatal(dst != -1, "Invalid connection."); auto dfs = only(kgraph.coordinates.getOutputNodeIndices(dst, CT::isEdge)); if(!dfs) continue; if(!std::holds_alternative(*addAssign->expression)) continue; return {{parent, *child}}; } return {}; } /** * @brief Reconnect incoming and outgoing DataFlow edges from dimension. * * Inputs coming into the dimension are augmented with `other_inputs`. */ void reflow(KernelGraph& graph, int dim, std::vector const& other_inputs) { namespace CT = rocRoller::KernelGraph::CoordinateGraph; std::vector inputs, outputs; for(auto const& tag : graph.coordinates.getNeighbours(dim)) { auto df = graph.coordinates.get(tag); if(!df) continue; auto parents = graph.coordinates.getNeighbours(tag); for(auto const parent : parents) inputs.push_back(parent); graph.coordinates.deleteElement(tag); } for(auto const& tag : graph.coordinates.getNeighbours(dim)) { auto df = graph.coordinates.get(tag); if(!df) continue; auto children = graph.coordinates.getNeighbours(tag); for(auto const child : children) outputs.push_back(child); graph.coordinates.deleteElement(tag); } std::copy(other_inputs.cbegin(), other_inputs.cend(), std::back_inserter(inputs)); graph.coordinates.addElement(CT::DataFlow(), inputs, outputs); } /** * @brief Fuse Add(Multiply(., .), .) into MultiplyAdd(., ., .). */ KernelGraph fuseMultiplyAdd(KernelGraph const& original) { using Expression::multiplyAdd; auto kgraph = original; std::unordered_set excluded; for(;;) { auto candidate = findMultiplyAdd(kgraph, excluded); if(!candidate) break; auto const& [multiplyTag, addTag] = *candidate; excluded.emplace(multiplyTag); auto [addLHS, addLHSDF] = getBinaryLHS(kgraph, addTag); auto mulDST = getDEST(kgraph, multiplyTag); if(addLHS != mulDST) continue; auto addDST = getDEST(kgraph, addTag); auto [addRHS, addRHSDF] = getBinaryRHS(kgraph, addTag); auto [mulLHS, mulLHSDF] = getBinaryLHS(kgraph, multiplyTag); auto [mulRHS, mulRHSDF] = getBinaryRHS(kgraph, multiplyTag); auto fma = multiplyAdd(mulLHSDF, mulRHSDF, addRHSDF); // Reuse register type and count from Multiply auto fmaAssign = *kgraph.control.get(multiplyTag); fmaAssign.expression = fma; auto fmaTag = kgraph.control.addElement(fmaAssign); // Connect FMA; delete Multiply and Add operations reconnect(kgraph, -1, multiplyTag); reconnect(kgraph, fmaTag, multiplyTag); reconnect(kgraph, fmaTag, addTag); reconnect(kgraph, fmaTag, addTag); kgraph.control.deleteElement(multiplyTag); kgraph.control.deleteElement(addTag); // Tidy up coordinate graph reflow(kgraph, addLHS, {addRHS}); kgraph.coordinates.deleteElement(addLHS); // Tidy up connections kgraph.mapper.purge(multiplyTag); kgraph.mapper.purge(addTag); // Connect FMA kgraph.mapper.connect(fmaTag, addDST, NaryArgument::DEST); } return kgraph; } KernelGraph FuseExpressions::apply(KernelGraph const& original) { return fuseMultiplyAdd(original); } }