// Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include #include "MiopenConvDescriptor.hpp" #include "MiopenConvPlanBuilder.hpp" #include "MiopenUtils.hpp" #include "engines/plans/MiopenConvBwdPlan.hpp" #include "engines/plans/MiopenConvFwdPlan.hpp" #include "engines/plans/MiopenConvWrwPlan.hpp" // These miopenConvolution*GetWorkSpaceSizeRange entry points are exported from // libMIOpen but intentionally absent from the public miopen.h, so we declare them // locally to call them. The signatures are copied verbatim from MIOpen, whose // no-op top-level `const` on the pointer-typedef parameters trips clang-tidy, so // we suppress those checks to keep the prototypes identical. // NOLINTBEGIN(misc-misplaced-const,readability-avoid-const-params-in-decls) extern "C" { miopenStatus_t miopenConvolutionForwardGetWorkSpaceSizeRange(miopenHandle_t handle, const miopenTensorDescriptor_t wDesc, const miopenTensorDescriptor_t xDesc, const miopenConvolutionDescriptor_t convDesc, const miopenTensorDescriptor_t yDesc, size_t* minWorkspaceSize, size_t* maxWorkspaceSize); miopenStatus_t miopenConvolutionBackwardDataGetWorkSpaceSizeRange(miopenHandle_t handle, const miopenTensorDescriptor_t dyDesc, const miopenTensorDescriptor_t wDesc, const miopenConvolutionDescriptor_t convDesc, const miopenTensorDescriptor_t dxDesc, size_t* minWorkspaceSize, size_t* maxWorkspaceSize); miopenStatus_t miopenConvolutionBackwardWeightsGetWorkSpaceSizeRange( miopenHandle_t handle, const miopenTensorDescriptor_t dyDesc, const miopenTensorDescriptor_t xDesc, const miopenConvolutionDescriptor_t convDesc, const miopenTensorDescriptor_t dwDesc, size_t* minWorkspaceSize, size_t* maxWorkspaceSize); } // NOLINTEND(misc-misplaced-const,readability-avoid-const-params-in-decls) namespace miopen_plugin { MiopenConvPlanBuilder::MiopenConvPlanBuilder(bool deterministic) : _deterministic(deterministic) { } namespace { bool isApplicableFwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); size_t solutionCount = 0; try { const ConvFwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); if(!params.validTensors()) { return false; } auto status = miopenConvolutionForwardGetSolutionCount(handle.miopenHandle, params.w().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.y().tensorDescriptor(), &solutionCount); if(status != miopenStatusSuccess) { return false; } } catch(const std::exception& e) { HIPDNN_PLUGIN_LOG_INFO(e.what()); return false; } return solutionCount != 0; } bool isApplicableBwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); size_t solutionCount = 0; try { const ConvBwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); if(!params.validTensors()) { return false; } auto status = miopenConvolutionBackwardDataGetSolutionCount(handle.miopenHandle, params.dy().tensorDescriptor(), params.w().tensorDescriptor(), params.conv().convDescriptor(), params.dx().tensorDescriptor(), &solutionCount); if(status != miopenStatusSuccess) { return false; } } catch(const std::exception& e) { HIPDNN_PLUGIN_LOG_INFO(e.what()); return false; } return solutionCount != 0; } bool isApplicableWrw(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); size_t solutionCount = 0; try { const ConvWrwParams params(attr, opGraph.getTensorMap(), deterministicEnabled); if(!params.validTensors()) { return false; } auto status = miopenConvolutionBackwardWeightsGetSolutionCount(handle.miopenHandle, params.dy().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.dw().tensorDescriptor(), &solutionCount); if(status != miopenStatusSuccess) { return false; } } catch(const std::exception& e) { HIPDNN_PLUGIN_LOG_INFO(e.what()); return false; } return solutionCount != 0; } MiopenConvPlanBuilder::WorkspaceSizeRange getWorkspaceSizeRangeFwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvFwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); size_t minWorkspace = 0; size_t maxWorkspace = 0; THROW_ON_MIOPEN_FAILURE( miopenConvolutionForwardGetWorkSpaceSizeRange(handle.miopenHandle, params.w().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.y().tensorDescriptor(), &minWorkspace, &maxWorkspace)); HIPDNN_PLUGIN_LOG_INFO("Convolution Fwd: Workspace range: min=" << minWorkspace << ", max=" << maxWorkspace); return {minWorkspace, maxWorkspace}; } MiopenConvPlanBuilder::WorkspaceSizeRange getWorkspaceSizeRangeBwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvBwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); size_t minWorkspace = 0; size_t maxWorkspace = 0; THROW_ON_MIOPEN_FAILURE( miopenConvolutionBackwardDataGetWorkSpaceSizeRange(handle.miopenHandle, params.dy().tensorDescriptor(), params.w().tensorDescriptor(), params.conv().convDescriptor(), params.dx().tensorDescriptor(), &minWorkspace, &maxWorkspace)); HIPDNN_PLUGIN_LOG_INFO("Convolution Bwd: Workspace range: min=" << minWorkspace << ", max=" << maxWorkspace); return {minWorkspace, maxWorkspace}; } MiopenConvPlanBuilder::WorkspaceSizeRange getWorkspaceSizeRangeWrw(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvWrwParams params(attr, opGraph.getTensorMap(), deterministicEnabled); size_t minWorkspace = 0; size_t maxWorkspace = 0; THROW_ON_MIOPEN_FAILURE( miopenConvolutionBackwardWeightsGetWorkSpaceSizeRange(handle.miopenHandle, params.dy().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.dw().tensorDescriptor(), &minWorkspace, &maxWorkspace)); HIPDNN_PLUGIN_LOG_INFO("Convolution Wrw: Workspace range: min=" << minWorkspace << ", max=" << maxWorkspace); return {minWorkspace, maxWorkspace}; } size_t getMaxWorkspaceSizeFwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, const HipdnnMiopenSettings& executionSettings, bool deterministicEnabled) { size_t workSpaceSize = executionSettings.selectedWorkspaceSize(); if(workSpaceSize == 0) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvFwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); THROW_ON_MIOPEN_FAILURE( miopenConvolutionForwardGetWorkSpaceSize(handle.miopenHandle, params.w().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.y().tensorDescriptor(), &workSpaceSize)); } return workSpaceSize; } size_t getMaxWorkspaceSizeBwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, const HipdnnMiopenSettings& executionSettings, bool deterministicEnabled) { size_t workSpaceSize = executionSettings.selectedWorkspaceSize(); if(workSpaceSize == 0) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvBwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); THROW_ON_MIOPEN_FAILURE( miopenConvolutionBackwardDataGetWorkSpaceSize(handle.miopenHandle, params.dy().tensorDescriptor(), params.w().tensorDescriptor(), params.conv().convDescriptor(), params.dx().tensorDescriptor(), &workSpaceSize)); } return workSpaceSize; } size_t getMaxWorkspaceSizeWrw(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, const HipdnnMiopenSettings& executionSettings, bool deterministicEnabled) { size_t workSpaceSize = executionSettings.selectedWorkspaceSize(); if(workSpaceSize == 0) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); const ConvWrwParams params(attr, opGraph.getTensorMap(), deterministicEnabled); THROW_ON_MIOPEN_FAILURE( miopenConvolutionBackwardWeightsGetWorkSpaceSize(handle.miopenHandle, params.dy().tensorDescriptor(), params.x().tensorDescriptor(), params.conv().convDescriptor(), params.dw().tensorDescriptor(), &workSpaceSize)); } return workSpaceSize; } void buildPlanFwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, HipdnnMiopenContext& executionContext, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); ConvFwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); auto plan = std::make_unique( handle, std::move(params), executionContext.executionSettings()); executionContext.setPlan(std::move(plan)); } void buildPlanBwd(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, HipdnnMiopenContext& executionContext, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); ConvBwdParams params(attr, opGraph.getTensorMap(), deterministicEnabled); auto plan = std::make_unique( handle, std::move(params), executionContext.executionSettings()); executionContext.setPlan(std::move(plan)); } void buildPlanWrw(const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, HipdnnMiopenContext& executionContext, bool deterministicEnabled) { const auto& attr = opGraph.getNodeWrapper(0) .attributesAs(); ConvWrwParams params(attr, opGraph.getTensorMap(), deterministicEnabled); auto plan = std::make_unique( handle, std::move(params), executionContext.executionSettings()); executionContext.setPlan(std::move(plan)); } } // namespace bool MiopenConvPlanBuilder::isApplicable( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph) const { // Execute-time override shapes can diverge from the compile-time dims this // builder matched exactly; the plan bakes those dims into the MIOpen // descriptors it builds, so decline rather than risk a mismatch (RFC 0008 §4.6). if(opGraph.getGraph().is_override_shape_enabled()) { HIPDNN_PLUGIN_LOG_INFO("Convolution plan builder does not support override shapes"); return false; } if(opGraph.nodeCount() != 1) { HIPDNN_PLUGIN_LOG_INFO( "Convolution plan builder is applicable only for single node graphs. Graph " "has " << opGraph.nodeCount() << " nodes"); return false; } if(opGraph.getNode(0).compute_data_type() != hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT) { HIPDNN_PLUGIN_LOG_ERROR("Convolution plan builder only supports nodes with an fp32 " "compute_data_type"); return false; } const auto& node = opGraph.getNode(0); bool ret = false; switch(node.attributes_type()) { case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionFwdAttributes: ret = isApplicableFwd(handle, opGraph, _deterministic); break; case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionBwdAttributes: ret = isApplicableBwd(handle, opGraph, _deterministic); break; case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionWrwAttributes: ret = isApplicableWrw(handle, opGraph, _deterministic); break; default: break; } if(!ret) { HIPDNN_PLUGIN_LOG_INFO("Convolution plan builder is not applicable for this graph"); } return ret; } MiopenConvPlanBuilder::WorkspaceSizeRange MiopenConvPlanBuilder::getWorkspaceSizeRange( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph) const { if(opGraph.nodeCount() != 1) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Convolution plan builder supports only single node graphs. Graph has " + std::to_string(opGraph.nodeCount()) + " nodes"); } const auto& node = opGraph.getNode(0); switch(node.attributes_type()) { case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionFwdAttributes: return getWorkspaceSizeRangeFwd(handle, opGraph, _deterministic); case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionBwdAttributes: return getWorkspaceSizeRangeBwd(handle, opGraph, _deterministic); case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionWrwAttributes: return getWorkspaceSizeRangeWrw(handle, opGraph, _deterministic); default: throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Unsupported node type for convolution plan builder: " + std::string( hipdnn_flatbuffers_sdk::data_objects::toString(node.attributes_type()))); } } size_t MiopenConvPlanBuilder::getMaxWorkspaceSize( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, const HipdnnMiopenSettings& executionSettings) const { if(opGraph.nodeCount() != 1) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Convolution plan builder supports only single node graphs. Graph has " + std::to_string(opGraph.nodeCount()) + " nodes"); } const auto& node = opGraph.getNode(0); switch(node.attributes_type()) { case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionFwdAttributes: return getMaxWorkspaceSizeFwd(handle, opGraph, executionSettings, _deterministic); case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionBwdAttributes: return getMaxWorkspaceSizeBwd(handle, opGraph, executionSettings, _deterministic); case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionWrwAttributes: return getMaxWorkspaceSizeWrw(handle, opGraph, executionSettings, _deterministic); default: throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Unsupported node type for convolution plan builder: " + std::string( hipdnn_flatbuffers_sdk::data_objects::toString(node.attributes_type()))); } } void MiopenConvPlanBuilder::initializeExecutionSettings( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IEngineConfig& engineConfig, HipdnnMiopenSettings& executionSettings) const { // Read workspace size limit knob setting if(engineConfig.isValid() && engineConfig.hasKnobSetting(hipdnn_plugin_sdk::WORKSPACE_SIZE_LIMIT_KNOB_NAME)) { const auto& knobSetting = engineConfig.getKnobSettingByName(hipdnn_plugin_sdk::WORKSPACE_SIZE_LIMIT_KNOB_NAME); if(knobSetting.valueType() != hipdnn_flatbuffers_sdk::data_objects::KnobValue::IntValue) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Workspace size limit knob setting value is not an integer. Type: " + std::string(hipdnn_flatbuffers_sdk::data_objects::EnumNameKnobValue( knobSetting.valueType()))); } auto value = knobSetting.valueAs().value(); if(value < 0) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_INVALID_VALUE, "Invalid workspace size limit value: " + std::to_string(value) + ". Must be >= 0"); } const auto range = getWorkspaceSizeRange(handle, opGraph); if(static_cast(value) < range.min || static_cast(value) > range.max) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_INVALID_VALUE, "Invalid workspace size limit value: " + std::to_string(value) + ". Must be in range [" + std::to_string(range.min) + ", " + std::to_string(range.max) + "]"); } executionSettings.setWorkspaceSizeLimit(static_cast(value)); executionSettings.setDefaultWorkspaceSize(range.max); } if(!executionSettings.workspaceSizeLimit().has_value()) { const auto maxWs = getMaxWorkspaceSize(handle, opGraph, executionSettings); executionSettings.setDefaultWorkspaceSize(maxWs); } } void MiopenConvPlanBuilder::buildPlan( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph, [[maybe_unused]] const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IEngineConfig& engineConfig, HipdnnMiopenContext& executionContext) const { if(opGraph.nodeCount() != 1) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Convolution plan builder supports only single node graphs. Graph has " + std::to_string(opGraph.nodeCount()) + " nodes"); } const auto& nodeWrapper = opGraph.getNodeWrapper(0); const auto nodeName = nodeWrapper.name(); switch(nodeWrapper.attributesType()) { case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionFwdAttributes: HIPDNN_PLUGIN_LOG_INFO("Building convolution fwd plan for node: " << nodeName); buildPlanFwd(handle, opGraph, executionContext, _deterministic); break; case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionBwdAttributes: HIPDNN_PLUGIN_LOG_INFO("Building convolution bwd plan for node: " << nodeName); buildPlanBwd(handle, opGraph, executionContext, _deterministic); break; case hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionWrwAttributes: HIPDNN_PLUGIN_LOG_INFO("Building convolution wrw plan for node: " << nodeName); buildPlanWrw(handle, opGraph, executionContext, _deterministic); break; default: throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_BAD_PARAM, "Unsupported node type for convolution plan builder: " + std::string( hipdnn_flatbuffers_sdk::data_objects::toString(nodeWrapper.attributesType()))); } } std::vector MiopenConvPlanBuilder::getCustomKnobs( const HipdnnMiopenHandle& handle, const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& opGraph) const { std::vector knobs; if(!isApplicable(handle, opGraph)) { return knobs; } // Workspace size limit knob const auto range = getWorkspaceSizeRange(handle, opGraph); // Validate that size_t values can fit into int64_t if(range.min > std::numeric_limits::max()) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, "Workspace size range minimum (" + std::to_string(range.min) + ") exceeds maximum representable int64_t value"); } if(range.max > std::numeric_limits::max()) { throw hipdnn_plugin_sdk::HipdnnPluginException( HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, "Workspace size range maximum (" + std::to_string(range.max) + ") exceeds maximum representable int64_t value"); } const auto minWorkspace = static_cast(range.min); const auto maxWorkspace = static_cast(range.max); hipdnn_flatbuffers_sdk::data_objects::KnobT workspaceKnob; workspaceKnob.knob_id = hipdnn_plugin_sdk::WORKSPACE_SIZE_LIMIT_KNOB_NAME; workspaceKnob.description = "Workspace size limit in bytes"; hipdnn_flatbuffers_sdk::data_objects::IntValueT workspaceDefaultValue; workspaceDefaultValue.value = maxWorkspace; workspaceKnob.default_value.Set(workspaceDefaultValue); hipdnn_flatbuffers_sdk::data_objects::IntConstraintT workspaceConstraint; workspaceConstraint.min_value = minWorkspace; workspaceConstraint.max_value = maxWorkspace; workspaceConstraint.step = 1; workspaceKnob.constraint.Set(workspaceConstraint); knobs.push_back(std::move(workspaceKnob)); return knobs; } } // namespace miopen_plugin