// Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #pragma once #include #include #include #include #include #include #include #include #include namespace hipdnn_gpu_ref { namespace detail { template inline std::vector buildConvDefines(bool useTf32 = false) { std::vector defines; defines.emplace_back(std::string("-DX_TYPE=") + HipRtcTypeName::VALUE); defines.emplace_back(std::string("-DW_TYPE=") + HipRtcTypeName::VALUE); defines.emplace_back(std::string("-DY_TYPE=") + HipRtcTypeName::VALUE); defines.emplace_back(std::string("-DCOMPUTE_TYPE=") + HipRtcTypeName::VALUE); if(useTf32) { defines.emplace_back("-DUSE_TF32"); } return defines; } } // namespace detail class GpuFpReferenceConvolution { public: // --- Forward convolution (fprop) --- // TensorBase overload — symmetric padding convenience wrapper. template static void fprop(hipdnn_data_sdk::utilities::TensorBase& x, hipdnn_data_sdk::utilities::TensorBase& w, hipdnn_data_sdk::utilities::TensorBase& y, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, double alpha = 1.0, double beta = 0.0, bool useTf32 = false) { fprop( x, w, y, convStrides, dilations, padding, padding, alpha, beta, useTf32); } // TensorBase overload — asymmetric padding. // Takes non-const references because deviceData() may trigger host→device sync. template static void fprop(hipdnn_data_sdk::utilities::TensorBase& x, hipdnn_data_sdk::utilities::TensorBase& w, hipdnn_data_sdk::utilities::TensorBase& y, const std::vector& convStrides, const std::vector& dilations, const std::vector& prePadding, const std::vector& postPadding, double alpha = 1.0, double beta = 0.0, bool useTf32 = false) { fprop(x.memory().deviceData(), w.memory().deviceData(), y.memory().deviceData(), x.dims(), w.dims(), y.dims(), x.strides(), w.strides(), y.strides(), convStrides, dilations, prePadding, postPadding, alpha, beta, useTf32); y.memory().markDeviceModified(); } // --- Backward data gradient (dgrad) --- // gradX and w are non-const because MigratableMemory::deviceData() triggers // lazy host-to-device synchronization, which mutates internal state. // Overload for uniform padding template static void dgrad(hipdnn_data_sdk::utilities::TensorBase& gradX, hipdnn_data_sdk::utilities::TensorBase& w, hipdnn_data_sdk::utilities::TensorBase& gradY, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, double alpha = 1.0, double beta = 0.0) { dgrad( gradX, w, gradY, convStrides, dilations, padding, padding, alpha, beta); } template static void dgrad(hipdnn_data_sdk::utilities::TensorBase& gradX, hipdnn_data_sdk::utilities::TensorBase& w, hipdnn_data_sdk::utilities::TensorBase& gradY, const std::vector& convStrides, const std::vector& dilations, const std::vector& prePadding, const std::vector& postPadding, double alpha = 1.0, double beta = 0.0) { validateInput( gradX.dims(), w.dims(), gradY.dims(), convStrides, dilations, prePadding, postPadding); const auto nDims = gradX.dims().size(); auto defines = detail::buildConvDefines(); auto* dxPtr = gradX.memory().deviceData(); auto* wPtr = w.memory().deviceData(); auto* dyPtr = gradY.memory().deviceData(); // Only prePadding is passed to the kernel. Post-padding is implicitly // handled by the output tensor dimensions and the kernel's bounds checks. if(nDims == 3) { launchDgrad1d(dxPtr, wPtr, dyPtr, gradX.dims(), w.dims(), gradY.dims(), gradX.strides(), w.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 4) { launchDgrad2d(dxPtr, wPtr, dyPtr, gradX.dims(), w.dims(), gradY.dims(), gradX.strides(), w.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 5) { launchDgrad3d(dxPtr, wPtr, dyPtr, gradX.dims(), w.dims(), gradY.dims(), gradX.strides(), w.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else { throw std::invalid_argument( "GPU kernels support 1D/2D/3D convolutions (tensor rank 3/4/5), got rank " + std::to_string(nDims)); } gradX.memory().markDeviceModified(); } // --- Backward weight gradient (wgrad) --- // x and gradY are non-const because MigratableMemory::deviceData() triggers // lazy host-to-device synchronization, which mutates internal state. // Overload for uniform padding template static void wgrad(hipdnn_data_sdk::utilities::TensorBase& x, hipdnn_data_sdk::utilities::TensorBase& gradW, hipdnn_data_sdk::utilities::TensorBase& gradY, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, double alpha = 1.0, double beta = 0.0) { wgrad( x, gradW, gradY, convStrides, dilations, padding, padding, alpha, beta); } template static void wgrad(hipdnn_data_sdk::utilities::TensorBase& x, hipdnn_data_sdk::utilities::TensorBase& gradW, hipdnn_data_sdk::utilities::TensorBase& gradY, const std::vector& convStrides, const std::vector& dilations, const std::vector& prePadding, const std::vector& postPadding, double alpha = 1.0, double beta = 0.0) { validateInput( x.dims(), gradW.dims(), gradY.dims(), convStrides, dilations, prePadding, postPadding); const auto nDims = x.dims().size(); auto defines = detail::buildConvDefines(); auto* xPtr = x.memory().deviceData(); auto* dwPtr = gradW.memory().deviceData(); auto* dyPtr = gradY.memory().deviceData(); // Only prePadding is passed to the kernel. Post-padding is implicitly // handled by the output tensor dimensions and the kernel's bounds checks. if(nDims == 3) { launchWgrad1d(xPtr, dwPtr, dyPtr, x.dims(), gradW.dims(), gradY.dims(), x.strides(), gradW.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 4) { launchWgrad2d(xPtr, dwPtr, dyPtr, x.dims(), gradW.dims(), gradY.dims(), x.strides(), gradW.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 5) { launchWgrad3d(xPtr, dwPtr, dyPtr, x.dims(), gradW.dims(), gradY.dims(), x.strides(), gradW.strides(), gradY.strides(), convStrides, dilations, prePadding, defines, alpha, beta); } else { throw std::invalid_argument( "GPU kernels support 1D/2D/3D convolutions (tensor rank 3/4/5), got rank " + std::to_string(nDims)); } gradW.memory().markDeviceModified(); } private: // --- Raw device-pointer fprop (implementation detail) --- template static void fprop(const void* xPtr, const void* wPtr, void* yPtr, const std::vector& xDims, const std::vector& wDims, const std::vector& yDims, const std::vector& xStrides, const std::vector& wStrides, const std::vector& yStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& prePadding, const std::vector& postPadding, double alpha = 1.0, double beta = 0.0, bool useTf32 = false) { validateInput(xDims, wDims, yDims, convStrides, dilations, prePadding, postPadding); const auto nDims = xDims.size(); auto defines = detail::buildConvDefines(useTf32); // Only prePadding is passed to the kernel. Post-padding is implicitly // handled by the output tensor dimensions and the kernel's bounds checks. if(nDims == 3) { launchFprop1d(xPtr, wPtr, yPtr, xDims, wDims, yDims, xStrides, wStrides, yStrides, convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 4) { launchFprop2d(xPtr, wPtr, yPtr, xDims, wDims, yDims, xStrides, wStrides, yStrides, convStrides, dilations, prePadding, defines, alpha, beta); } else if(nDims == 5) { launchFprop3d(xPtr, wPtr, yPtr, xDims, wDims, yDims, xStrides, wStrides, yStrides, convStrides, dilations, prePadding, defines, alpha, beta); } else { throw std::invalid_argument( "GPU kernels support 1D/2D/3D convolutions (tensor rank 3/4/5), got rank " + std::to_string(nDims)); } } // --- Validation --- static void validateInput(const std::vector& xDims, const std::vector& wDims, const std::vector& yDims, const std::vector& strides, const std::vector& dilations, const std::vector& prePadding, const std::vector& postPadding) { const auto nDims = xDims.size(); // GPU kernels only support 1D (3), 2D (4), and 3D (5) convolutions if(nDims != 3 && nDims != 4 && nDims != 5) { throw std::invalid_argument("Input tensor must have 3 dimensions (1D conv), " "4 dimensions (2D conv), or 5 dimensions (3D conv)"); } hipdnn_test_sdk::utilities::validateConvolutionParams( xDims, wDims, yDims, strides, dilations, prePadding, postPadding); } // --- Kernel launchers (defined in GpuFpReferenceConvolution.cpp) --- static void launchFprop1d(const void* xPtr, const void* wPtr, void* yPtr, const std::vector& xDims, const std::vector& wDims, const std::vector& yDims, const std::vector& xTensorStrides, const std::vector& wTensorStrides, const std::vector& yTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchFprop2d(const void* xPtr, const void* wPtr, void* yPtr, const std::vector& xDims, const std::vector& wDims, const std::vector& yDims, const std::vector& xTensorStrides, const std::vector& wTensorStrides, const std::vector& yTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchFprop3d(const void* xPtr, const void* wPtr, void* yPtr, const std::vector& xDims, const std::vector& wDims, const std::vector& yDims, const std::vector& xTensorStrides, const std::vector& wTensorStrides, const std::vector& yTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); // --- Dgrad kernel launchers (defined in GpuFpReferenceConvolution.cpp) --- static void launchDgrad1d(void* dxPtr, const void* wPtr, const void* dyPtr, const std::vector& dxDims, const std::vector& wDims, const std::vector& dyDims, const std::vector& dxTensorStrides, const std::vector& wTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchDgrad2d(void* dxPtr, const void* wPtr, const void* dyPtr, const std::vector& dxDims, const std::vector& wDims, const std::vector& dyDims, const std::vector& dxTensorStrides, const std::vector& wTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchDgrad3d(void* dxPtr, const void* wPtr, const void* dyPtr, const std::vector& dxDims, const std::vector& wDims, const std::vector& dyDims, const std::vector& dxTensorStrides, const std::vector& wTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); // --- Wgrad kernel launchers (defined in GpuFpReferenceConvolution.cpp) --- static void launchWgrad1d(const void* xPtr, void* dwPtr, const void* dyPtr, const std::vector& xDims, const std::vector& dwDims, const std::vector& dyDims, const std::vector& xTensorStrides, const std::vector& dwTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchWgrad2d(const void* xPtr, void* dwPtr, const void* dyPtr, const std::vector& xDims, const std::vector& dwDims, const std::vector& dyDims, const std::vector& xTensorStrides, const std::vector& dwTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); static void launchWgrad3d(const void* xPtr, void* dwPtr, const void* dyPtr, const std::vector& xDims, const std::vector& dwDims, const std::vector& dyDims, const std::vector& xTensorStrides, const std::vector& dwTensorStrides, const std::vector& dyTensorStrides, const std::vector& convStrides, const std::vector& dilations, const std::vector& padding, const std::vector& defines, double alpha, double beta); }; } // namespace hipdnn_gpu_ref