// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include namespace rocRoller::KernelGraph { namespace LoadPackedDetail { std::map getStaticCoords(int controlNode, KernelGraph const& graph) { std::map rv; // Map from coord to the node that we used to set that coord, so // that we can tell if we are above or below there. std::map coordSetBy; auto setCoordNodes = getContainingSetCoordinates(graph, controlNode); for(auto setCoordNode : setCoordNodes) { auto setCoord = graph.control.get(setCoordNode).value(); auto connections = graph.mapper.getConnections(setCoordNode); AssertFatal(connections.size() == 1, "Invalid SetCoordinate operation; coordinate missing."); auto coord = connections[0].coordinate; auto iter = coordSetBy.find(coord); if(iter == coordSetBy.end()) { AssertFatal(!rv.contains(coord)); rv[coord] = setCoord.value; coordSetBy[coord] = setCoordNode; } else { auto ordering = graph.control.compareNodes( rocRoller::UpdateCache, setCoordNode, iter->second); if(ordering == ControlGraph::NodeOrdering::LeftInBodyOfRight) { Log::debug("{}: replacing {} with {}.", coord, toString(rv[coord]), toString(setCoord.value)); rv[coord] = setCoord.value; iter->second = setCoordNode; } } } return rv; } std::map fillRegisterCoords(std::unordered_set const& requiredCoords, KernelGraph const& graph, ContextPtr context) { using namespace CoordinateGraph; std::map rv; auto isRegisterDim = [](auto dim) -> bool { using T = std::decay_t; return CIsAnyOf; }; for(auto coord : requiredCoords) { if(std::visit(isRegisterDim, graph.coordinates.getNode(coord))) { auto reg = Register::Value::Placeholder( context, Register::Type::Vector, DataType::Int32, 1); rv[coord] = reg->expression(); } } return rv; } std::tuple> getFakeTransformerForControlNode( int controlNode, KernelGraph const& graph, ContextPtr context) { auto const& [coords, paths] = findAllRequiredCoordinates(controlNode, graph); auto staticCoords = getStaticCoords(controlNode, graph); auto regCoords = fillRegisterCoords(coords, graph, context); CoordinateGraph::Transformer xform(&graph.coordinates); for(auto const& [coord, expr] : staticCoords) xform.setCoordinate(coord, expr); for(auto const& [coord, expr] : regCoords) { xform.setCoordinate(coord, expr); } std::set remainingCoords; for(auto coord : coords) if(!xform.hasCoordinate(coord)) remainingCoords.insert(coord); return {xform, remainingCoords}; } int getFastestMovingCoord(std::set const& coords, KernelGraph const& graph) { AssertFatal(!coords.empty()); if(coords.size() == 1) return *coords.begin(); std::optional fastestCoord; std::optional maxSubDim; for(auto coord : coords) { auto dim = graph.coordinates.getNode(coord); auto visitor = rocRoller::overloaded{ [&] T>(T const& subdim) { if(!maxSubDim || subdim.dim > maxSubDim) { fastestCoord = coord; maxSubDim = subdim.dim; } }, [&](CoordinateGraph::VGPR const& vgpr) { fastestCoord = coord; maxSubDim = 1000; }, [&](auto const& other) { Log::debug("Other: {}", toString(other)); }}; std::visit(visitor, dim); } return fastestCoord.value(); } } LoadPacked::LoadPacked(ContextPtr context) : m_context(context) { } KernelGraph LoadPacked::apply(KernelGraph const& original) { using namespace ControlGraph; using namespace CoordinateGraph; using namespace LoadPackedDetail; std::set changedDims; auto rv = original; auto pred = [&](int node) { auto visitor = [](auto op) -> bool { using T = std::decay_t; if constexpr(CIsAnyOf) { return DataTypeInfo::Get(op.varType).packedVariableType().has_value(); } return false; }; return rv.control.getElementType(node) == Graph::ElementType::Node && std::visit(visitor, rv.control.getNode(node)); }; auto roots = rv.control.roots().to(); for(auto node : rv.control.findNodes(roots, pred)) { Log::debug("Load/store op: ({})", node); auto [target, direction] = getOperationTarget(node, rv); target = getTransformTarget(target, rv); auto op = rv.control.getNode(node); auto varType = getVariableType(op); auto packedType = DataTypeInfo::Get(varType).packedVariableType().value(); auto packedInfo = DataTypeInfo::Get(packedType); auto waveTileTag = rv.mapper.get(node); if(waveTileTag > 0) { auto waveTile = rv.coordinates.get(waveTileTag); auto elements = waveTile.value().elements(); auto [_, lane] = rv.getDimension(node); auto activeLanesInWave = getUnsignedInt(evaluate(lane.size)); AssertFatal(elements % activeLanesInWave == 0); auto elementsPerThread = elements / activeLanesInWave; if(elementsPerThread < packedInfo.packing) { Log::debug( "Skipping node {} due to insufficient elementsPerThread: {} (packing {})", node, elementsPerThread, packedInfo.packing); continue; } } if(rv.mapper.get(node, 0) > 0 && rv.mapper.get(node, 1) > 0) { auto [elemXTag, elemX] = rv.getDimension(node, 0); auto [elemYTag, elemY] = rv.getDimension(node, 1); auto const m = getUnsignedInt(evaluate(elemX.size)); auto n = getUnsignedInt(evaluate(elemY.size)); if(m * n < packedInfo.packing) { Log::debug("Skipping node {}: {}x{} due to ElementNumber size.", node, ShowValue(m), ShowValue(n)); continue; } } auto [xform, remainingCoords] = getFakeTransformerForControlNode(node, rv, m_context); Log::debug("Remaining coords: {}", concatenate(remainingCoords)); if(remainingCoords.empty()) continue; auto fastestCoord = getFastestMovingCoord(remainingCoords, rv); for(auto coord : remainingCoords) { xform.setCoordinate(coord, Expression::literal(0u)); } Log::debug("Fastest coord: {}, Target: {}, looking {}", fastestCoord, target, toString(direction)); auto exprs = direction != Graph::Direction::Upstream ? xform.reverseStride(fastestCoord, Expression::literal(1u), {target}) : xform.forwardStride(fastestCoord, Expression::literal(1u), {target}); Log::debug("Stride: {}", toString(exprs[0])); if(evaluationTimes(exprs[0])[Expression::EvaluationTime::Translate]) { Log::debug("Stride: {}", toString(evaluate(exprs[0]))); if(getUnsignedInt(evaluate(exprs[0])) == 1) { Log::debug("Setting {} varType from {} to {}.", node, toString(varType), toString(packedType)); setVariableType(op, packedType); rv.control.setElement(node, op); } } } return rv; } }