// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include namespace rocRoller::KernelGraph::ControlGraph { std::unordered_map> ControlGraph::nodeOrderTable() const { populateOrderCache(); std::unordered_map> table; for(auto const& [node, orders] : m_orderCache) { for(int other : orders.after) { if(node < other) table[node][other] = NodeOrdering::LeftFirst; } for(int other : orders.inBody) { if(node < other) table[node][other] = NodeOrdering::RightInBodyOfLeft; else table[other][node] = NodeOrdering::LeftInBodyOfRight; } } return table; } std::string ControlGraph::nodeOrderTableString(std::set const& nodes) const { populateOrderCache(); if(nodes.empty()) { return "Empty order cache.\n"; } std::ostringstream msg; int width = std::ceil(std::log10(static_cast(*nodes.rbegin()))); width = std::max(width, 3); msg << std::setw(width) << " " << "\\"; for(int n : nodes) msg << " " << std::setw(width) << n; for(int i : nodes) { msg << std::endl << std::setw(width) << i << "|"; for(int j : nodes) { if(i == j) { msg << " "; auto oldFill = msg.fill('-'); msg << std::setw(width) << "-"; msg.fill(oldFill); } else { msg << " " << std::setw(width) << abbrev(lookupOrder(CacheOnly, i, j)); } } msg << " | " << std::setw(width) << i; } msg << std::endl << std::setw(width) << " " << "|"; for(int n : nodes) msg << " " << std::setw(width) << n; msg << std::endl; return msg.str(); } std::string ControlGraph::nodeOrderTableString() const { populateOrderCache(); TIMER(t, "nodeOrderTable"); std::set nodes; for(auto const& [node, orders] : m_orderCache) { if(!orders.after.empty() or !orders.before.empty() or !orders.inBody.empty() or !orders.containing.empty()) { nodes.insert(node); } for(int n : orders.after) nodes.insert(n); for(int n : orders.before) nodes.insert(n); for(int n : orders.inBody) nodes.insert(n); for(int n : orders.containing) nodes.insert(n); } return nodeOrderTableString(nodes); } void ControlGraph::clearCache(Graph::GraphModification modification) { Hypergraph::clearCache(modification); if(modification == Graph::GraphModification::AddElement && m_cacheStatus != CacheStatus::Invalid) { // If adding a new element and order is non-empty (partial or valid) m_cacheStatus = CacheStatus::Partial; } else { m_orderCache.clear(); m_cacheStatus = CacheStatus::Invalid; } m_descendentCache.clear(); } bool ControlGraph::isModificationAllowed(int index) const { if(not Settings::getInstance()->get(Settings::EnforceGraphConstraints)) return true; if(not m_changesRestricted) return true; auto const& el = getElement(index); if(std::holds_alternative(el)) { return std::visit( [](auto&& arg) { using OpType = std::decay_t; return !( std::is_same_v or std::is_same_v); }, std::get(el)); } else { // // Theoretically, add/delete Body edge should be disallowed. But sometimes // delete Body edges is OK (e.g., Simplify), and currently there is no way // to know if this is called in a valid or invalid use case. // return true; } } void ControlGraph::sortOrderCache() const { int const maxId = std::ranges::max(m_orderCache | std::views::keys); std::vector> bits((maxId + 64) / 64); auto check = [&](const auto&... vec) { ( [&]() { for(auto v : vec) { int const id = v / 64; int const remain = v & 63; AssertFatal(!bits[id][remain], "A node has two orders", ShowValue(id), ShowValue(remain), ShowValue(v)); bits[id].set(remain); } }(), ...); std::ranges::fill(bits, std::bitset<64>{}); }; for(auto& [node, orders] : m_orderCache) { std::ranges::sort(orders.after); std::ranges::sort(orders.before); std::ranges::sort(orders.inBody); std::ranges::sort(orders.containing); check(orders.after, orders.before, orders.inBody, orders.containing); } } void ControlGraph::populateOrderCache() const { TIMER(t, "populateOrderCache"); if(m_cacheStatus == CacheStatus::Valid) return; m_orderCache.clear(); auto rootNodes = roots().to(); populateOrderCache(rootNodes); sortOrderCache(); m_cacheStatus = CacheStatus::Valid; // // m_descendentCache is only used to help build m_orderCache, // and it must be cleared after finish building m_orderCache // to ensure no stale data being used when building m_orderCache // next time. // m_descendentCache.clear(); } template Range> std::vector ControlGraph::populateOrderCache(Range const& startingNodes) const { std::vector rv; for(auto it = startingNodes.begin(); it != startingNodes.end(); ++it) { auto nodes = populateOrderCache(*it); rv.insert(rv.end(), nodes.begin(), nodes.end()); } return rv; } std::vector ControlGraph::populateOrderCache(int startingNode) const { auto ccEntry = m_descendentCache.find(startingNode); if(ccEntry != m_descendentCache.end()) return ccEntry->second; static_assert(std::variant_size_v == 5, "Currently the available edge types are Sequence(0), Initialize(1), " "ForLoopIncrement(2), Body(3) and Else(4)." "If more edge types are added, this function has to be updated."); using GD = Graph::Direction; // Edge variant indices: Sequence(0), Initialize(1), ForLoopIncrement(2), Body(3), Else(4) std::array, std::variant_size_v> directChildren; for(auto edge : getNeighbours(startingNode)) { auto edgeTypeIndex = getEdge(edge).index(); for(auto child : getNeighbours(edge)) directChildren[edgeTypeIndex].push_back(child); } auto addDescendents = [this](std::vector const& children) -> std::vector { auto descendents = populateOrderCache(children); std::vector result; result.reserve(children.size() + descendents.size()); result.insert(result.end(), children.begin(), children.end()); result.insert(result.end(), descendents.begin(), descendents.end()); std::ranges::sort(result); result.erase(std::unique(result.begin(), result.end()), result.end()); return result; }; // Index: Initialize(1), Body(3), Else(4), ForLoopIncrement(2), Sequence(0) auto initNodes = addDescendents(directChildren[1]); auto bodyNodes = addDescendents(directChildren[3]); auto elseNodes = addDescendents(directChildren[4]); auto incNodes = addDescendents(directChildren[2]); auto seqNodes = addDescendents(directChildren[0]); // {init, body, else, inc} nodes are in the body of the current node writeOrderCache({startingNode}, initNodes, NodeOrdering::RightInBodyOfLeft); writeOrderCache({startingNode}, bodyNodes, NodeOrdering::RightInBodyOfLeft); writeOrderCache({startingNode}, elseNodes, NodeOrdering::RightInBodyOfLeft); writeOrderCache({startingNode}, incNodes, NodeOrdering::RightInBodyOfLeft); // Sequence connected nodes are after the current node writeOrderCache({startingNode}, seqNodes, NodeOrdering::LeftFirst); // {body, else, inc, sequence} are after init nodes writeOrderCache(initNodes, bodyNodes, NodeOrdering::LeftFirst); writeOrderCache(initNodes, elseNodes, NodeOrdering::LeftFirst); writeOrderCache(initNodes, incNodes, NodeOrdering::LeftFirst); writeOrderCache(initNodes, seqNodes, NodeOrdering::LeftFirst); // {else, inc, sequence} are after body nodes writeOrderCache(bodyNodes, elseNodes, NodeOrdering::LeftFirst); writeOrderCache(bodyNodes, incNodes, NodeOrdering::LeftFirst); writeOrderCache(bodyNodes, seqNodes, NodeOrdering::LeftFirst); // {inc, sequence} are after else nodes writeOrderCache(elseNodes, incNodes, NodeOrdering::LeftFirst); writeOrderCache(elseNodes, seqNodes, NodeOrdering::LeftFirst); // sequence are after inc nodes writeOrderCache(incNodes, seqNodes, NodeOrdering::LeftFirst); std::vector allNodes; allNodes.reserve(initNodes.size() + bodyNodes.size() + elseNodes.size() + incNodes.size() + seqNodes.size()); allNodes.insert(allNodes.end(), initNodes.begin(), initNodes.end()); allNodes.insert(allNodes.end(), bodyNodes.begin(), bodyNodes.end()); allNodes.insert(allNodes.end(), elseNodes.begin(), elseNodes.end()); allNodes.insert(allNodes.end(), incNodes.begin(), incNodes.end()); allNodes.insert(allNodes.end(), seqNodes.begin(), seqNodes.end()); std::ranges::sort(allNodes); allNodes.erase(std::unique(allNodes.begin(), allNodes.end()), allNodes.end()); m_descendentCache[startingNode] = allNodes; return allNodes; } template ARange, CForwardRangeOf BRange> void ControlGraph::writeOrderCache(ARange const& nodesA, BRange const& nodesB, NodeOrdering order) const { if(nodesA.size() == 0 or nodesB.size() == 0) return; auto selectVec = [](NodeOrders& orders, NodeOrdering order) -> std::vector& { switch(order) { case NodeOrdering::LeftFirst: return orders.after; case NodeOrdering::RightFirst: return orders.before; case NodeOrdering::RightInBodyOfLeft: return orders.inBody; case NodeOrdering::LeftInBodyOfRight: return orders.containing; default: break; } AssertFatal(false, "Invalid order: ", ShowValue(order)); return orders.after; // this statement should never be reached }; for(int a : nodesA) { auto& vec = selectVec(m_orderCache[a], order); vec.insert(vec.end(), nodesB.begin(), nodesB.end()); } auto oppositeOrder = opposite(order); for(int b : nodesB) { auto& vec = selectVec(m_orderCache[b], oppositeOrder); vec.insert(vec.end(), nodesA.begin(), nodesA.end()); } } void ControlGraph::writeOrderCache(int nodeA, int nodeB, NodeOrdering order) const { switch(order) { case NodeOrdering::LeftFirst: m_orderCache[nodeA].after.push_back(nodeB); m_orderCache[nodeB].before.push_back(nodeA); break; case NodeOrdering::RightFirst: m_orderCache[nodeA].before.push_back(nodeB); m_orderCache[nodeB].after.push_back(nodeA); break; case NodeOrdering::RightInBodyOfLeft: m_orderCache[nodeA].inBody.push_back(nodeB); m_orderCache[nodeB].containing.push_back(nodeA); break; case NodeOrdering::LeftInBodyOfRight: m_orderCache[nodeA].containing.push_back(nodeB); m_orderCache[nodeB].inBody.push_back(nodeA); break; default: break; } } NodeOrdering ControlGraph::lookupOrder(IgnoreCachePolicy const, int nodeA, int nodeB) const { TIMER(t, "ControlGraph::lookupOrder"); using GD = Graph::Direction; std::unordered_set visited_nodes; // Decide order of A and B when A is parent of B auto const getOrderIfParent = [&](int edge) { return std::holds_alternative(getEdge(edge)) ? NodeOrdering::LeftFirst : NodeOrdering::RightInBodyOfLeft; }; // Decide order of A and B when A and B are descendants of a node // And order of edge type : // Initialize -> Body -> Else -> ForLoopIncrement -> Sequence. auto const getOrderOfDescendants = [&](int edgeA, int edgeB) { auto edgeAElem = getEdge(edgeA); auto edgeBElem = getEdge(edgeB); AssertFatal(edgeAElem.index() != edgeBElem.index(), "edgeA and edgeB types should not be the same"); return std::visit( rocRoller::overloaded{ [&](auto const&) { AssertFatal(false, "Unhandled edge type in getOrderOfDescendants"); return NodeOrdering::Undefined; }, [&](Body const&) { return std::holds_alternative(edgeBElem) ? opposite(NodeOrdering::LeftFirst) : NodeOrdering::LeftFirst; }, [&](Sequence const&) { return opposite(NodeOrdering::LeftFirst); }, [&](Initialize const&) { return NodeOrdering::LeftFirst; }, [&](ForLoopIncrement const&) { return std::holds_alternative(edgeBElem) ? NodeOrdering::LeftFirst : opposite(NodeOrdering::LeftFirst); }, [&](Else const&) { return std::holds_alternative(edgeBElem) || std::holds_alternative(edgeBElem) ? opposite(NodeOrdering::LeftFirst) : NodeOrdering::LeftFirst; }}, edgeAElem); }; // {key, value} = {node index, edge index} std::unordered_map A_ancestors; // stack std::vector stk{nodeA}; // Traverse upstream from A to collect all ancestors of A while(!stk.empty()) { auto node = stk.back(); stk.pop_back(); for(auto edge : getNeighbours(node)) { for(auto parent : getNeighbours(edge)) { if(parent == nodeB) return opposite(getOrderIfParent(edge)); A_ancestors.insert({parent, edge}); if(!visited_nodes.contains(parent)) { visited_nodes.insert(parent); stk.push_back(parent); } } } } AssertFatal(stk.empty()); stk.push_back(nodeB); visited_nodes.clear(); // Traverse upstream from B to find common ancestors to decide order while(!stk.empty()) { auto node = stk.back(); stk.pop_back(); for(auto edge : getNeighbours(node)) { for(auto parent : getNeighbours(edge)) { if(parent == nodeA) return getOrderIfParent(edge); if(A_ancestors.contains(parent)) { // If this is a common ancestor, compare the types of both edges to // know the order if(getEdge(A_ancestors.at(parent)).index() != getEdge(edge).index()) { return getOrderOfDescendants(A_ancestors.at(parent), edge); } } if(!visited_nodes.contains(parent)) { visited_nodes.insert(parent); stk.push_back(parent); } } } } return NodeOrdering::Undefined; } static void validateNodes(ControlGraph const& control, int nodeA, int nodeB) { AssertFatal(nodeA != nodeB, ShowValue(nodeA)); AssertFatal(control.getElementType(nodeA) == Graph::ElementType::Node && control.getElementType(nodeB) == Graph::ElementType::Node, ShowValue(control.getElementType(nodeA)), ShowValue(control.getElementType(nodeB))); } NodeOrdering ControlGraph::compareNodes(CacheOnlyPolicy const, int nodeA, int nodeB) const { AssertFatal(m_cacheStatus == CacheStatus::Valid); validateNodes(*this, nodeA, nodeB); return lookupOrder(CacheOnly, nodeA, nodeB); } NodeOrdering ControlGraph::compareNodes(UpdateCachePolicy const, int nodeA, int nodeB) const { if(m_cacheStatus != CacheStatus::Valid) populateOrderCache(); return compareNodes(CacheOnly, nodeA, nodeB); } NodeOrdering ControlGraph::compareNodes(UseCacheIfAvailablePolicy const, int nodeA, int nodeB) const { validateNodes(*this, nodeA, nodeB); return m_cacheStatus == CacheStatus::Valid ? compareNodes(CacheOnly, nodeA, nodeB) : compareNodes(IgnoreCache, nodeA, nodeB); } NodeOrdering ControlGraph::compareNodes(IgnoreCachePolicy const, int nodeA, int nodeB) const { validateNodes(*this, nodeA, nodeB); return lookupOrder(IgnoreCache, nodeA, nodeB); } }