// Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #pragma once #include "Node.hpp" #include #include #include #include #include #include #include namespace hipdnn_frontend::graph { class SdpaFwdNode : public BaseNode { public: SdpaAttributes attributes; SdpaFwdNode(SdpaAttributes&& sdpaAttributes, const GraphAttributes& graphAttrs) : BaseNode(graphAttrs) , attributes(std::move(sdpaAttributes)) { } Error pre_validate_node() const override { HIPDNN_RETURN_IF_TRUE(attributes.hasUnsupportedUsage(), ErrorCode::INVALID_VALUE, attributes.getUnsupportedReason()); const auto q = attributes.get_q(); const auto k = attributes.get_k(); const auto v = attributes.get_v(); const auto o = attributes.get_o(); // Validate required tensors are present HIPDNN_RETURN_IF_FALSE( q, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing Q input")); HIPDNN_RETURN_IF_FALSE( k, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing K input")); HIPDNN_RETURN_IF_FALSE( v, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing V input")); HIPDNN_RETURN_IF_FALSE( o, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing O output")); const auto& qDims = q->get_dim(); const auto& kDims = k->get_dim(); const auto& vDims = v->get_dim(); // Rule 1: Q, K, V must be exactly rank-4 const size_t reqRank = 4; HIPDNN_RETURN_IF_NE(qDims.size(), reqRank, ErrorCode::INVALID_VALUE, "SdpaFwdNode: Query tensor Q must be rank-4, got rank=" + std::to_string(qDims.size())); HIPDNN_RETURN_IF_NE(kDims.size(), reqRank, ErrorCode::INVALID_VALUE, "SdpaFwdNode: Key tensor K must be rank-4, got rank=" + std::to_string(kDims.size())); HIPDNN_RETURN_IF_NE(vDims.size(), reqRank, ErrorCode::INVALID_VALUE, "SdpaFwdNode: Value tensor V must be rank-4, got rank=" + std::to_string(vDims.size())); HIPDNN_RETURN_IF_NE(qDims[0], kDims[0], ErrorCode::INVALID_VALUE, "SdpaFwdNode: batch size mismatch between Q and K: " + std::to_string(qDims[0]) + " vs " + std::to_string(kDims[0])); HIPDNN_RETURN_IF_NE(qDims[0], vDims[0], ErrorCode::INVALID_VALUE, "SdpaFwdNode: batch size mismatch between Q and V: " + std::to_string(qDims[0]) + " vs " + std::to_string(vDims[0])); // Rule 2: head_dim: Q[-1] == K[-1]; V[-1] is independent (may differ) const auto headDimQ = qDims[3]; const auto headDimK = kDims[3]; HIPDNN_RETURN_IF_NE(headDimQ, headDimK, ErrorCode::INVALID_VALUE, "SdpaFwdNode: head_dim mismatch between Q and K: " + std::to_string(headDimQ) + " vs " + std::to_string(headDimK)); // Rule 3: seq_kv: K[-2] == V[-2] const auto seqKvK = kDims[2]; const auto seqKvV = vDims[2]; HIPDNN_RETURN_IF_NE(seqKvK, seqKvV, ErrorCode::INVALID_VALUE, "SdpaFwdNode: seq_kv mismatch between K and V: " + std::to_string(seqKvK) + " vs " + std::to_string(seqKvV)); // Rule 4: num_heads % K_heads == 0; num_heads % V_heads == 0 (GQA/MQA) const auto numHeads = qDims[1]; const auto numHeadsK = kDims[1]; const auto numHeadsV = vDims[1]; HIPDNN_RETURN_IF_TRUE(numHeadsK <= 0, ErrorCode::INVALID_VALUE, "SdpaFwdNode: num_heads_k must be positive, got " + std::to_string(numHeadsK)); HIPDNN_RETURN_IF_TRUE(numHeads % numHeadsK != 0, ErrorCode::INVALID_VALUE, "SdpaFwdNode: num_heads must be divisible by num_heads_k for " "GQA/MQA. num_heads=" + std::to_string(numHeads) + ", num_heads_k=" + std::to_string(numHeadsK)); HIPDNN_RETURN_IF_TRUE(numHeadsV <= 0, ErrorCode::INVALID_VALUE, "SdpaFwdNode: num_heads_v must be positive, got " + std::to_string(numHeadsV)); HIPDNN_RETURN_IF_TRUE(numHeads % numHeadsV != 0, ErrorCode::INVALID_VALUE, "SdpaFwdNode: num_heads must be divisible by num_heads_v for " "GQA/MQA. num_heads=" + std::to_string(numHeads) + ", num_heads_v=" + std::to_string(numHeadsV)); // Rule 5: Optional attention mask validation const auto attnMask = attributes.get_bias(); if(attnMask) { const auto& maskDims = attnMask->get_dim(); const auto maskRank = maskDims.size(); HIPDNN_RETURN_IF_TRUE(maskRank > static_cast(4), ErrorCode::INVALID_VALUE, "SdpaFwdNode: ATTN_MASK rank must be <= 4, got rank=" + std::to_string(maskRank)); const auto seqQ = qDims[2]; const auto seqKv = kDims[2]; const auto maskLast = maskDims[maskRank - 1]; HIPDNN_RETURN_IF_TRUE(maskLast != seqKv && maskLast != 1, ErrorCode::INVALID_VALUE, "SdpaFwdNode: ATTN_MASK last dim must equal seq_kv (" + std::to_string(seqKv) + ") or 1, got " + std::to_string(maskLast)); if(maskRank >= 2) { const auto maskSecondLast = maskDims[maskRank - 2]; HIPDNN_RETURN_IF_TRUE(maskSecondLast != seqQ && maskSecondLast != 1, ErrorCode::INVALID_VALUE, "SdpaFwdNode: ATTN_MASK second-to-last dim must equal " "seq_q (" + std::to_string(seqQ) + ") or 1, got " + std::to_string(maskSecondLast)); } } // Rule 6: attn_scale may be given as a tensor or a baked value, not both const auto scale = attributes.get_attn_scale(); HIPDNN_RETURN_IF_TRUE( scale && attributes.attn_scale_value.has_value(), ErrorCode::INVALID_VALUE, "SdpaFwdNode: attn_scale tensor and attn_scale_value cannot both be set"); // Rule 6b: Optional scale must be a scalar tensor (volume == 1) if(scale) { HIPDNN_CHECK_ERROR(detail::validateScalarParameter(scale, "SCALE tensor")); } // Rule 7: If O has dims already set, validate them const auto& oDims = o->get_dim(); if(!oDims.empty()) { const auto headDimV = vDims[3]; HIPDNN_RETURN_IF_NE(oDims.size(), reqRank, ErrorCode::INVALID_VALUE, "SdpaFwdNode: Output O must be rank-4, got rank=" + std::to_string(oDims.size())); HIPDNN_RETURN_IF_NE(oDims[0], qDims[0], ErrorCode::INVALID_VALUE, "SdpaFwdNode: Output O batch mismatch: expected " + std::to_string(qDims[0]) + " got " + std::to_string(oDims[0])); HIPDNN_RETURN_IF_NE(oDims[1], qDims[1], ErrorCode::INVALID_VALUE, "SdpaFwdNode: Output O num_heads mismatch: expected " + std::to_string(qDims[1]) + " got " + std::to_string(oDims[1])); HIPDNN_RETURN_IF_NE(oDims[2], qDims[2], ErrorCode::INVALID_VALUE, "SdpaFwdNode: Output O seq_q mismatch: expected " + std::to_string(qDims[2]) + " got " + std::to_string(oDims[2])); HIPDNN_RETURN_IF_NE(oDims[3], headDimV, ErrorCode::INVALID_VALUE, "SdpaFwdNode: Output O head_dim must match V head_dim: expected " + std::to_string(headDimV) + " got " + std::to_string(oDims[3])); } return {}; } Error infer_properties_node() override { const auto q = attributes.get_q(); const auto v = attributes.get_v(); const auto o = attributes.get_o(); HIPDNN_RETURN_IF_FALSE( q, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing Q input")); HIPDNN_RETURN_IF_FALSE( v, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing V input")); HIPDNN_RETURN_IF_FALSE( o, ErrorCode::ATTRIBUTE_NOT_SET, std::string("SdpaFwdNode missing O output")); HIPDNN_CHECK_ERROR(attributes.fill_from_context(graph_attributes)); // Output O shape: [batch, num_heads, seq_q, head_dim_v] // head_dim_v (V's last dim) may differ from head_dim_qk if(o->get_dim().empty()) { const auto& qDims = q->get_dim(); const auto& vDims = v->get_dim(); o->set_dim({qDims[0], qDims[1], qDims[2], vDims[3]}); } if(o->get_stride().empty()) { auto oStrides = hipdnn_data_sdk::utilities::generateStrides(o->get_dim()); o->set_stride(oStrides); } // Stats output: softmax logsumexp values, shape [batch, num_heads, seq_q, 1] const auto stats = attributes.get_stats(); if(stats) { if(stats->get_dim().empty()) { const auto& qDims = q->get_dim(); stats->set_dim({qDims[0], qDims[1], qDims[2], 1}); } if(stats->get_stride().empty()) { auto statsStrides = hipdnn_data_sdk::utilities::generateStrides(stats->get_dim()); stats->set_stride(statsStrides); } } return {}; } Error unpack_from_descriptor( hipdnnBackendDescriptor_t opDesc, std::unordered_map>& tensorMap) override { SdpaAttributes attrs; HIPDNN_CHECK_ERROR(detail::unpackSdpaFwdOperation(opDesc, tensorMap, attrs)); attributes = std::move(attrs); return {}; } Error create_operation( std::unordered_map& tensorDescs, std::vector& operations) const override { return detail::createSdpaFwdOperation(attributes, tensorDescs, operations); } }; } // namespace hipdnn_frontend::graph