derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
995 lines
44 KiB
C++
995 lines
44 KiB
C++
/*******************************************************************************
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (c) 2018 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in all
|
|
* copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*
|
|
*******************************************************************************/
|
|
#pragma once
|
|
#include <array>
|
|
#include <iostream>
|
|
#include <iterator>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <utility>
|
|
#include <miopen/miopen.h>
|
|
#include <miopen/convolution.hpp>
|
|
#include <miopen/batch_norm.hpp>
|
|
#include <miopen/activ.hpp>
|
|
#include <miopen/tensor.hpp>
|
|
#include <miopen/fusion_plan.hpp>
|
|
#include "get_handle.hpp"
|
|
#include "tensor_holder.hpp"
|
|
#include "verify.hpp"
|
|
|
|
template <class T>
|
|
void convHostForward(const tensor<T>& input,
|
|
tensor<T>& output,
|
|
const tensor<T>& weights,
|
|
const int bias_mode,
|
|
const tensor<T>& bias,
|
|
const miopenConvolutionDescriptor_t convDesc)
|
|
{
|
|
|
|
int in_n, in_c, in_h, in_w;
|
|
int in_nstride, in_cstride, in_hstride, in_wstride;
|
|
std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(input.desc.GetLengths());
|
|
std::tie(in_nstride, in_cstride, in_hstride, in_wstride) =
|
|
miopen::tien<4>(input.desc.GetStrides());
|
|
|
|
int wei_n, wei_c, wei_h, wei_w;
|
|
int wei_nstride, wei_cstride, wei_hstride, wei_wstride;
|
|
std::tie(wei_n, wei_c, wei_h, wei_w) = miopen::tien<4>(weights.desc.GetLengths());
|
|
std::tie(wei_nstride, wei_cstride, wei_hstride, wei_wstride) =
|
|
miopen::tien<4>(weights.desc.GetStrides());
|
|
|
|
int out_n, out_c, out_h, out_w;
|
|
int out_nstride, out_cstride, out_hstride, out_wstride;
|
|
std::tie(out_n, out_c, out_h, out_w) = miopen::tien<4>(output.desc.GetLengths());
|
|
std::tie(out_nstride, out_cstride, out_hstride, out_wstride) =
|
|
miopen::tien<4>(output.desc.GetStrides());
|
|
|
|
int stride_h, stride_w, pad_h, pad_w, dilation_h, dilation_w;
|
|
miopenConvolutionMode_t mode;
|
|
miopenPaddingMode_t pmode = miopen::deref(convDesc).paddingMode;
|
|
miopenGetConvolutionDescriptor(
|
|
convDesc, &mode, &pad_h, &pad_w, &stride_h, &stride_w, &dilation_h, &dilation_w);
|
|
|
|
if(pmode == miopenPaddingSame)
|
|
{
|
|
pad_h = (in_h % stride_h == 0) ? (std::max((wei_h - stride_h), 0))
|
|
: (std::max((wei_h - (in_h % stride_h)), 0));
|
|
pad_w = (in_w % stride_w == 0) ? (std::max((wei_w - stride_w), 0))
|
|
: (std::max((wei_w - (in_w % stride_w)), 0));
|
|
pad_h /= 2;
|
|
pad_w /= 2;
|
|
}
|
|
else if(pmode == miopenPaddingValid)
|
|
{
|
|
pad_h = 0;
|
|
pad_w = 0;
|
|
}
|
|
|
|
if(out_h <= 0 || out_w <= 0)
|
|
MIOPEN_THROW("Invalid Test Case: Check Output Dimension.");
|
|
|
|
for(int o = 0; o < out_n; o++)
|
|
{ // mini-batch size
|
|
for(int w = 0; w < out_c; w++)
|
|
{ // out_channels (num filters)
|
|
for(int i = 0; i < out_h; i++)
|
|
{ // output_height (from getforwardoutputdim())
|
|
int in_off_h = i * stride_h;
|
|
for(int j = 0; j < out_w; j++)
|
|
{ // output_width (from getforwardoutputdim())
|
|
/*auto acc = static_cast<T>(0.);*/
|
|
auto acc = static_cast<double>(0.);
|
|
int in_off_w = j * stride_w;
|
|
for(int k = 0; k < in_c; k++)
|
|
{ // in_channels (RGB)
|
|
for(int x = 0; x < wei_h; x++)
|
|
{
|
|
int in_x = in_off_h - pad_h + x * dilation_h;
|
|
if(in_x >= 0 && in_x < in_h)
|
|
{
|
|
for(int y = 0; y < wei_w; y++)
|
|
{
|
|
int in_y = in_off_w - pad_w + y * dilation_w;
|
|
if(in_y >= 0 && in_y < in_w)
|
|
{
|
|
acc += double(
|
|
static_cast<T>(input[o * in_nstride + k * in_cstride +
|
|
in_x * in_w + in_y]) *
|
|
static_cast<T>(weights(w, k, x, y)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
acc = bias_mode != 0 ? acc + static_cast<double>(bias[w]) : acc;
|
|
output[o * out_nstride + w * out_cstride + i * out_hstride + j] =
|
|
static_cast<T>(acc);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
template <class T, class Tref, class U, class V = U>
|
|
void batchNormSpatialHostInference(const tensor<T>& input,
|
|
tensor<Tref>& output,
|
|
const tensor<U>& scale,
|
|
const tensor<U>& bias,
|
|
double epsilon,
|
|
const tensor<V>& estimatedMean,
|
|
const tensor<V>& estimatedVariance,
|
|
bool useInverseVariance = false)
|
|
{
|
|
|
|
int n_batches, channels, height, width;
|
|
std::tie(n_batches, channels, height, width) = miopen::tien<4>(input.desc.GetLengths());
|
|
miopen::par_for(channels, 1, [&](int cidx) { // via channel
|
|
V mean = estimatedMean(0, cidx, 0, 0);
|
|
V variance = estimatedVariance(0, cidx, 0, 0);
|
|
double invertVar =
|
|
useInverseVariance ? static_cast<double>(variance) : 1.0 / sqrt(variance + epsilon);
|
|
// process the batch per channel
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batches; bidx++)
|
|
{ // via mini_batch
|
|
double elemStd = static_cast<double>(input(bidx, cidx, row, column)) - mean;
|
|
double inhat = elemStd * invertVar;
|
|
output(bidx, cidx, row, column) =
|
|
static_cast<T>(scale(0, cidx, 0, 0) * inhat + bias(0, cidx, 0, 0));
|
|
// printf("output: %f\n",scale(0, cidx, 0, 0) * inhat + bias(0, cidx, 0, 0));
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
template <class T, class U, class V, class Tref>
|
|
void batchNormPerActivHostInference(const tensor<T>& input,
|
|
tensor<Tref>& output,
|
|
const tensor<U>& scale,
|
|
const tensor<U>& bias,
|
|
double epsilon,
|
|
const tensor<V>& estimatedMean,
|
|
const tensor<V>& estimatedVariance,
|
|
bool useInverseVariance = false)
|
|
{
|
|
int n_batches, channels, height, width;
|
|
std::tie(n_batches, channels, height, width) = miopen::tien<4>(input.desc.GetLengths());
|
|
miopen::par_for(channels, 1, [&](int cidx) { // via channel
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
// apply down the n_batch dimension
|
|
double mean = estimatedMean(0, cidx, row, column);
|
|
double variance = estimatedVariance(0, cidx, row, column);
|
|
double elemInvVar = useInverseVariance ? variance : 1.0 / sqrt(variance + epsilon);
|
|
for(int bidx = 0; bidx < n_batches; bidx++)
|
|
{ // via mini_batch
|
|
// per (x-dims) channel load a block of data into LDS
|
|
double elemStd = input(bidx, cidx, row, column) - mean;
|
|
double inhat = elemStd * elemInvVar;
|
|
output(bidx, cidx, row, column) =
|
|
scale(0, cidx, row, column) * inhat + bias(0, cidx, row, column);
|
|
// printf("output: %f\n",output(bidx, cidx, row, column));
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
template <class T, class U, class Tref = U, class Tout>
|
|
void batchNormSpatialHostFwdTrain(const tensor<T>& input,
|
|
tensor<Tout>& out,
|
|
const tensor<U>& scale,
|
|
const tensor<U>& bias,
|
|
double epsilon,
|
|
double expAvgFactor,
|
|
tensor<Tref>& saveMean,
|
|
tensor<Tref>& saveInvVar,
|
|
tensor<Tref>& runMean,
|
|
tensor<Tref>& runVar)
|
|
{
|
|
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(input.desc.GetLengths());
|
|
const auto nhw = double(height * width * n_batch);
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double elemStd = 0.;
|
|
double variance_accum = 0.;
|
|
double mean_accum = 0.;
|
|
double invVar = 0.;
|
|
double newRunMean = 0.;
|
|
double adjust = 0.;
|
|
|
|
// process the batch per channel
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
// #1 calculate the mean
|
|
// iterating through the stack of images in the mini_batch
|
|
auto inval = static_cast<double>(input(bidx, cidx, row, column));
|
|
mean_accum += inval;
|
|
variance_accum += inval * inval;
|
|
} // end for (column)
|
|
} // end for (row)
|
|
} // end for (n)
|
|
|
|
mean_accum /= nhw;
|
|
variance_accum /= nhw;
|
|
variance_accum += (-mean_accum * mean_accum);
|
|
invVar = 1.0 / sqrt(variance_accum + epsilon);
|
|
|
|
// #4 apply the normalization
|
|
// x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
// #5 Gamma and Beta adjust
|
|
// y_i = gamma*x_hat + beta
|
|
elemStd = (static_cast<double>(input(bidx, cidx, row, column)) -
|
|
mean_accum); // (x_i - mean)
|
|
out(bidx, cidx, row, column) = static_cast<T>(
|
|
scale(0, cidx, 0, 0) * (invVar * elemStd) + bias(0, cidx, 0, 0));
|
|
} // for (column)
|
|
} // for (row)
|
|
} // end for(n_batchs)
|
|
if(!saveMean.data.empty())
|
|
{
|
|
saveMean(0, cidx, 0, 0) = mean_accum;
|
|
saveInvVar(0, cidx, 0, 0) = invVar;
|
|
}
|
|
if(!runMean.data.empty())
|
|
{
|
|
newRunMean = runMean(0, cidx, 0, 0) * (1 - expAvgFactor);
|
|
runMean(0, cidx, 0, 0) = mean_accum * expAvgFactor + newRunMean; // newMean*factor + tmp
|
|
// var(n+1) = p * var(n-1) + (1 - p)*(b/b-1)*var(n)
|
|
adjust = (n_batch * height * width == 1) ? variance_accum
|
|
: (nhw / (nhw - 1)) * variance_accum;
|
|
runVar(0, cidx, 0, 0) =
|
|
(1 - expAvgFactor) * runVar(0, cidx, 0, 0) + expAvgFactor * adjust;
|
|
}
|
|
});
|
|
}
|
|
|
|
template <typename XDataType,
|
|
typename DxDataType,
|
|
typename DyDataType,
|
|
typename ScaleDataType,
|
|
typename AccDataType,
|
|
typename RefDataType>
|
|
void batchNormSpatialHostBwdTrain(const tensor<XDataType>& x_input,
|
|
tensor<DyDataType>& dy_input,
|
|
tensor<DxDataType>& dx_out,
|
|
const tensor<ScaleDataType>& bnScale,
|
|
const tensor<ScaleDataType>& bnBias,
|
|
tensor<RefDataType>& dscale,
|
|
tensor<RefDataType>& dbias,
|
|
const tensor<AccDataType>& savedMean,
|
|
const tensor<AccDataType>& savedInvVar,
|
|
miopenActivationMode_t activ_mode,
|
|
double activ_beta,
|
|
double activ_alpha)
|
|
{
|
|
double activ_gamma = 0.;
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(x_input.desc.GetLengths());
|
|
auto nhw = double(height * width * n_batch);
|
|
int in_cstride = height * width;
|
|
|
|
if(activ_mode > 0)
|
|
{
|
|
tensor<AccDataType> input_norm =
|
|
tensor<AccDataType>{x_input.desc.GetLayoutEnum().value(), x_input.desc.GetLengths()};
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double mean = 0.0;
|
|
double invVar = 0.0;
|
|
double elemStd = 0.;
|
|
double mean_accum = 0.0;
|
|
double variance_accum = 0.0;
|
|
if(!savedMean.data.empty())
|
|
{
|
|
mean = static_cast<double>(savedMean(0, cidx, 0, 0)); // HxW elements
|
|
invVar = static_cast<double>(savedInvVar(0, cidx, 0, 0)); // HxW elements
|
|
}
|
|
else
|
|
{
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
auto inval = static_cast<double>(x_input(bidx, cidx, row, column));
|
|
mean_accum += inval;
|
|
variance_accum += inval * inval;
|
|
}
|
|
}
|
|
}
|
|
mean_accum /= nhw;
|
|
variance_accum /= nhw;
|
|
variance_accum += (-mean_accum * mean_accum);
|
|
mean = mean_accum;
|
|
invVar = 1.0 / sqrt(variance_accum);
|
|
}
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
elemStd = static_cast<double>(x_input(bidx, cidx, row, column)) -
|
|
mean; // (x_i - mean)
|
|
input_norm(bidx, cidx, row, column) = static_cast<AccDataType>(
|
|
bnScale(0, cidx, 0, 0) * (elemStd * invVar) + bnBias(0, cidx, 0, 0));
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
activationHostBnormBwd(activ_mode,
|
|
activ_gamma,
|
|
activ_beta,
|
|
activ_alpha,
|
|
dy_input.data,
|
|
input_norm.data,
|
|
dy_input.data);
|
|
}
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double elemStd = 0.;
|
|
unsigned int xhat_index;
|
|
double mean = 0.0;
|
|
double invVar = 0.0;
|
|
double dyelem = 0.;
|
|
std::vector<double> xhat(static_cast<std::size_t>(n_batch) * in_cstride, 0.0);
|
|
// process the batch per channel
|
|
dscale(0, cidx, 0, 0) = 0.;
|
|
dbias(0, cidx, 0, 0) = 0.;
|
|
|
|
if(!savedMean.data.empty())
|
|
{
|
|
|
|
mean = savedMean(0, cidx, 0, 0); // HxW elements
|
|
invVar = savedInvVar(0, cidx, 0, 0); // HxW elements
|
|
}
|
|
else
|
|
{
|
|
double variance_accum = 0.;
|
|
double mean_accum = 0.;
|
|
double inv_Var = 0.;
|
|
|
|
// process the batch per channel
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
// #1 calculate the mean
|
|
// iterating through the stack of images in the mini_batch
|
|
auto inval = static_cast<double>(x_input(bidx, cidx, row, column));
|
|
mean_accum += inval;
|
|
variance_accum += inval * inval;
|
|
} // end for (column)
|
|
} // end for (row)
|
|
} // end for (n)
|
|
|
|
mean_accum /= nhw;
|
|
variance_accum /= nhw;
|
|
variance_accum += (-mean_accum * mean_accum);
|
|
inv_Var = 1.0 / sqrt(variance_accum);
|
|
|
|
mean = mean_accum;
|
|
invVar = inv_Var;
|
|
}
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
// per (x-dims) channel load a block of data into LDS
|
|
elemStd = static_cast<double>(x_input(bidx, cidx, row, column)) -
|
|
mean; // (x_i - mean)
|
|
xhat[xhat_index] = elemStd * invVar;
|
|
dyelem = static_cast<double>(dy_input(bidx, cidx, row, column));
|
|
dbias(0, cidx, 0, 0) += dyelem;
|
|
dscale(0, cidx, 0, 0) += xhat[xhat_index] * dyelem;
|
|
} // end for(n_batch)
|
|
} // for (column)
|
|
} // for (row)
|
|
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
|
|
double tmp1 = nhw * dy_input(bidx, cidx, row, column) - dbias(0, cidx, 0, 0);
|
|
double tmp2 = -xhat[xhat_index] * dscale(0, cidx, 0, 0);
|
|
double tmp3 = (bnScale(0, cidx, 0, 0) * invVar) / nhw;
|
|
dx_out(bidx, cidx, row, column) =
|
|
static_cast<RefDataType>(tmp3 * (tmp2 + tmp1));
|
|
} // end for(n_batchs)
|
|
} // for (column)
|
|
} // for (row)
|
|
}); // for (channel)
|
|
}
|
|
|
|
template <typename XDataType,
|
|
typename DyDataType,
|
|
typename DxDataType,
|
|
typename ScaleDataType,
|
|
typename AccDataType,
|
|
typename OutRefDataType,
|
|
typename RefDataType>
|
|
void batchNormActivSpatialHostBwdTrain(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const tensor<XDataType>& x_input,
|
|
const tensor<DyDataType>& dy_input,
|
|
const tensor<DxDataType>& y_input,
|
|
tensor<OutRefDataType>& dx_out,
|
|
const tensor<ScaleDataType>& bnScale,
|
|
const tensor<AccDataType>& bias,
|
|
tensor<RefDataType>& dscale,
|
|
tensor<RefDataType>& dbias,
|
|
const tensor<AccDataType>& savedMean,
|
|
const tensor<AccDataType>& savedInvVar)
|
|
{
|
|
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(x_input.desc.GetLengths());
|
|
auto nhw = double(height * width * n_batch);
|
|
int in_cstride = height * width;
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double elemStd = 0.;
|
|
unsigned int xhat_index;
|
|
double mean = static_cast<double>(savedMean(0, cidx, 0, 0)); // HxW elements
|
|
double invVar = static_cast<double>(savedInvVar(0, cidx, 0, 0)); // HxW elements
|
|
double dyelem = 0.;
|
|
std::vector<double> xhat(static_cast<std::size_t>(n_batch) * in_cstride, 0.0);
|
|
// process the batch per channel
|
|
dscale(0, cidx, 0, 0) = 0.;
|
|
dbias(0, cidx, 0, 0) = 0.;
|
|
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
|
|
// recompute forward batch norm
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
// per (x-dims) channel load a block of data into LDS
|
|
elemStd = static_cast<double>(x_input(bidx, cidx, row, column)) -
|
|
mean; // (x_i - mean)
|
|
xhat[xhat_index] = elemStd * invVar;
|
|
double bnrefowd =
|
|
bnScale(0, cidx, 0, 0) * xhat[xhat_index] + bias(0, cidx, 0, 0);
|
|
activationHostBwdElement(activMode,
|
|
gamma,
|
|
beta,
|
|
alpha,
|
|
dy_input(bidx, cidx, row, column),
|
|
bnrefowd,
|
|
y_input(bidx, cidx, row, column),
|
|
dyelem);
|
|
dbias(0, cidx, 0, 0) += dyelem;
|
|
dscale(0, cidx, 0, 0) += xhat[xhat_index] * dyelem;
|
|
} // end for(n_batch)
|
|
} // for (column)
|
|
} // for (row)
|
|
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
double bnrefowd =
|
|
bnScale(0, cidx, 0, 0) * xhat[xhat_index] + bias(0, cidx, 0, 0);
|
|
activationHostBwdElement(activMode,
|
|
gamma,
|
|
beta,
|
|
alpha,
|
|
dy_input(bidx, cidx, row, column),
|
|
bnrefowd,
|
|
y_input(bidx, cidx, row, column),
|
|
dyelem);
|
|
// double tmp1 = nhw * dy_input(bidx, cidx, row, column) - dbias(0, cidx, 0, 0);
|
|
double tmp1 = nhw * dyelem - dbias(0, cidx, 0, 0);
|
|
double tmp2 = -xhat[xhat_index] * dscale(0, cidx, 0, 0);
|
|
double tmp3 = (bnScale(0, cidx, 0, 0) * invVar) / nhw;
|
|
dx_out(bidx, cidx, row, column) = static_cast<DxDataType>(tmp3 * (tmp2 + tmp1));
|
|
} // end for(n_batchs)
|
|
} // for (column)
|
|
} // for (row)
|
|
}); // for (channel)
|
|
}
|
|
|
|
template <class T, class U, class Tref, class TOutref>
|
|
void batchNormPerActHostFwdTrain(const tensor<T>& input,
|
|
tensor<TOutref>& out,
|
|
const tensor<U>& scale,
|
|
const tensor<U>& bias,
|
|
double epsilon,
|
|
double expAvgFactor,
|
|
tensor<Tref>& saveMean,
|
|
tensor<Tref>& saveInvVar,
|
|
tensor<Tref>& runMean,
|
|
tensor<Tref>& runVar)
|
|
{
|
|
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(input.desc.GetLengths());
|
|
const auto n = double(n_batch);
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double mean_accum = 0.;
|
|
double variance_accum = 0.;
|
|
double elemStd = 0.;
|
|
double elemInvVar = 0.;
|
|
double inhat = 0.;
|
|
double newRunMean = 0.;
|
|
double adjust = 0.;
|
|
|
|
// process the batch per channel
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
|
|
mean_accum = 0.;
|
|
variance_accum = 0.;
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
// #1 calculate the mean :: iterating through the stack of images in the
|
|
// mini_batch
|
|
auto intval = static_cast<double>(input(bidx, cidx, row, column));
|
|
mean_accum += intval;
|
|
variance_accum += intval * intval;
|
|
}
|
|
mean_accum /= n;
|
|
variance_accum /= n;
|
|
variance_accum = variance_accum - (mean_accum * mean_accum);
|
|
elemInvVar = 1.0 / double(sqrt(variance_accum + epsilon));
|
|
|
|
// #4 apply the normalization :: x_hat = (x_i - mean) / sqrt(variance_accum -
|
|
// epsilon)
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
elemStd = (input(bidx, cidx, row, column) - mean_accum); // (x_i - mean)
|
|
inhat = elemStd * elemInvVar;
|
|
// #5 Gamma and Beta adjust :: y_i = gamma*x_hat + beta
|
|
out(bidx, cidx, row, column) = static_cast<Tref>(
|
|
scale(0, cidx, row, column) * inhat + bias(0, cidx, row, column));
|
|
} // end for(n_batch)
|
|
|
|
if(!runMean.data.empty())
|
|
{
|
|
newRunMean = runMean(0, cidx, row, column) * (1.0 - expAvgFactor);
|
|
runMean(0, cidx, row, column) =
|
|
mean_accum * expAvgFactor + newRunMean; // newMean*factor + tmp
|
|
}
|
|
// var(n+1) = p * var(n-1) + (1 - p)*(b/b-1)*var(n)
|
|
if(!runVar.data.empty())
|
|
{
|
|
adjust = (n_batch == 1) ? variance_accum : (n / (n - 1.0)) * variance_accum;
|
|
runVar(0, cidx, row, column) =
|
|
(1 - expAvgFactor) * runVar(0, cidx, row, column) + expAvgFactor * adjust;
|
|
}
|
|
if(!saveMean.data.empty() || !saveInvVar.data.empty())
|
|
{
|
|
saveMean(0, cidx, row, column) = static_cast<Tref>(mean_accum);
|
|
saveInvVar(0, cidx, row, column) = static_cast<Tref>(elemInvVar);
|
|
}
|
|
|
|
} // for (column)
|
|
} // for (row)
|
|
});
|
|
}
|
|
|
|
template <typename XDataType,
|
|
typename DxDataType,
|
|
typename DyDataType = XDataType,
|
|
typename ScaleDataType,
|
|
typename AccDataType = ScaleDataType,
|
|
typename RefDataType = DxDataType>
|
|
void batchNormPerActHostBwdTrain(const tensor<XDataType>& x_input,
|
|
const tensor<DyDataType>& dy_input,
|
|
tensor<DxDataType>& dx_out,
|
|
const tensor<ScaleDataType>& scale,
|
|
tensor<RefDataType>& dscale,
|
|
tensor<RefDataType>& dbias,
|
|
const tensor<AccDataType>& savedMean,
|
|
const tensor<AccDataType>& savedInvVar)
|
|
{
|
|
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(x_input.desc.GetLengths());
|
|
int in_cstride = height * width;
|
|
auto n = double(n_batch);
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double elemStd = 0.;
|
|
unsigned int xhat_index;
|
|
double mean = 0.;
|
|
double elemInvVar = 0.;
|
|
double dyelem = 0.;
|
|
double dxhat = 0.;
|
|
double dxhathat = 0.;
|
|
double tmp1 = 0.;
|
|
std::vector<double> xhat(static_cast<std::size_t>(n_batch) * in_cstride);
|
|
|
|
// process the batch per channel
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
dxhat = 0.;
|
|
dxhathat = 0.;
|
|
|
|
if(!savedMean.data.empty())
|
|
{
|
|
mean = savedMean(0, cidx, row, column); // HxW elements
|
|
elemInvVar = savedInvVar(0, cidx, row, column); // HxW elements
|
|
}
|
|
else
|
|
{
|
|
double variance_accum = 0.;
|
|
double mean_accum = 0.;
|
|
|
|
// process the batch per channel
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
auto inval = static_cast<double>(x_input(bidx, cidx, row, column));
|
|
mean_accum += inval;
|
|
variance_accum += inval * inval;
|
|
} // end for (n)
|
|
|
|
mean_accum /= n;
|
|
variance_accum /= n;
|
|
variance_accum += (-mean_accum * mean_accum);
|
|
|
|
mean = mean_accum;
|
|
elemInvVar = 1.0 / sqrt(variance_accum);
|
|
}
|
|
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
// per (x-dims) channel load a block of data into LDS
|
|
elemStd = static_cast<double>(x_input(bidx, cidx, row, column)) -
|
|
mean; // (x_i - mean)
|
|
xhat[xhat_index] = elemStd * elemInvVar;
|
|
dyelem = static_cast<double>(dy_input(bidx, cidx, row, column));
|
|
dbias(0, cidx, row, column) += dyelem;
|
|
dscale(0, cidx, row, column) += xhat[xhat_index] * dyelem;
|
|
tmp1 = scale(0, cidx, row, column) * dyelem;
|
|
dxhat += tmp1;
|
|
dxhathat += tmp1 * xhat[xhat_index];
|
|
|
|
} // end for(n_batchs)
|
|
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
tmp1 = xhat[xhat_index] * dxhathat + dxhat;
|
|
double tmp2 =
|
|
n_batch * scale(0, cidx, row, column) * dy_input(bidx, cidx, row, column) -
|
|
tmp1;
|
|
double tmp3 = elemInvVar / (double(n));
|
|
dx_out(bidx, cidx, row, column) = static_cast<DxDataType>(tmp3 * tmp2);
|
|
} // end for(n_batchs)
|
|
} // for (column)
|
|
} // for (row)
|
|
});
|
|
}
|
|
|
|
template <class T, class U>
|
|
void batchNormActivPerActHostBwdTrain(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const tensor<T>& x_input,
|
|
const tensor<T>& dy_input,
|
|
const tensor<T>& y_input,
|
|
tensor<T>& dx_out,
|
|
const tensor<U>& scale,
|
|
const tensor<U>& bias,
|
|
tensor<U>& dscale,
|
|
tensor<U>& dbias,
|
|
const tensor<U>& savedMean,
|
|
const tensor<U>& savedInvVar)
|
|
{
|
|
|
|
int height, width, n_batch, channels;
|
|
std::tie(n_batch, channels, height, width) = miopen::tien<4>(x_input.desc.GetLengths());
|
|
int in_cstride = height * width;
|
|
auto n = double(n_batch);
|
|
|
|
miopen::par_for(channels, 1, [&](int cidx) {
|
|
double elemStd = 0.;
|
|
unsigned int xhat_index;
|
|
double mean = 0.;
|
|
double elemInvVar = 0.;
|
|
double dyelem = 0.;
|
|
double dxhat = 0.;
|
|
double dxhathat = 0.;
|
|
double tmp1 = 0.;
|
|
std::vector<double> xhat(static_cast<std::size_t>(n_batch) * in_cstride);
|
|
|
|
// process the batch per channel
|
|
for(int row = 0; row < height; row++)
|
|
{ // via rows
|
|
for(int column = 0; column < width; column++)
|
|
{ // via columns
|
|
dxhat = 0.;
|
|
dxhathat = 0.;
|
|
|
|
mean = savedMean(0, cidx, row, column); // HxW elements
|
|
elemInvVar = savedInvVar(0, cidx, row, column); // HxW elements
|
|
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
// per (x-dims) channel load a block of data into LDS
|
|
elemStd = static_cast<double>(x_input(bidx, cidx, row, column)) -
|
|
mean; // (x_i - mean)
|
|
xhat[xhat_index] = elemStd * elemInvVar;
|
|
double bnrefowd =
|
|
scale(0, cidx, row, column) * xhat[xhat_index] + bias(0, cidx, row, column);
|
|
activationHostBwdElement(activMode,
|
|
gamma,
|
|
beta,
|
|
alpha,
|
|
dy_input(bidx, cidx, row, column),
|
|
bnrefowd,
|
|
y_input(bidx, cidx, row, column),
|
|
dyelem);
|
|
/*dyelem = static_cast<double>(dy_input(bidx, cidx, row, column));*/
|
|
dbias(0, cidx, row, column) += dyelem;
|
|
dscale(0, cidx, row, column) += xhat[xhat_index] * dyelem;
|
|
tmp1 = scale(0, cidx, row, column) * dyelem;
|
|
dxhat += tmp1;
|
|
dxhathat += tmp1 * xhat[xhat_index];
|
|
|
|
} // end for(n_batchs)
|
|
|
|
for(int bidx = 0; bidx < n_batch; bidx++)
|
|
{ // via mini_batch
|
|
xhat_index = in_cstride * bidx + (width * row + column);
|
|
tmp1 = xhat[xhat_index] * dxhathat + dxhat;
|
|
double bnrefowd =
|
|
scale(0, cidx, row, column) * xhat[xhat_index] + bias(0, cidx, row, column);
|
|
activationHostBwdElement(activMode,
|
|
gamma,
|
|
beta,
|
|
alpha,
|
|
dy_input(bidx, cidx, row, column),
|
|
bnrefowd,
|
|
y_input(bidx, cidx, row, column),
|
|
dyelem);
|
|
double tmp2 = (n_batch * scale(0, cidx, row, column) * dyelem) - tmp1;
|
|
double tmp3 = elemInvVar / (double(n));
|
|
dx_out(bidx, cidx, row, column) = static_cast<T>(tmp3 * tmp2);
|
|
} // end for(n_batchs)
|
|
} // for (column)
|
|
} // for (row)
|
|
});
|
|
}
|
|
|
|
template <class F>
|
|
void visitActivationHostInfer(
|
|
miopenActivationMode_t activMode, double gamma, double beta, double alpha, F f)
|
|
{
|
|
switch(activMode)
|
|
{
|
|
case miopenActivationPASTHRU: // x
|
|
f([=](double x) { return x; });
|
|
break;
|
|
case miopenActivationLOGISTIC: // 1 / (1 + e^-x) //Sigmoid
|
|
f([=](double x) { return (1. / (1. + std::exp(-x))); });
|
|
break;
|
|
case miopenActivationTANH: // beta * tanh(alpha * x)
|
|
f([=](double x) { return (beta * std::tanh(alpha * x)); });
|
|
break;
|
|
case miopenActivationRELU: // max(0, x)
|
|
f([=](double x) { return ((x > 0.) ? x : 0.); });
|
|
break;
|
|
case miopenActivationSOFTRELU: // log(1 + e^x) // bonomial normal log likelihood
|
|
f([=](double x) {
|
|
return (x > 0.) ? (x + std::log1p(std::exp(-x))) : (std::log1p(std::exp(x)));
|
|
});
|
|
break;
|
|
case miopenActivationABS: // abs(x)
|
|
f([=](double x) { return (std::fabs(x)); });
|
|
break;
|
|
case miopenActivationPOWER: // (alpha + beta * x) ^ gamma
|
|
f([=](double x) {
|
|
auto v = (alpha + beta * x);
|
|
return (v <= std::numeric_limits<double>::epsilon()) ? 0. : pow(v, gamma);
|
|
});
|
|
break;
|
|
case miopenActivationCLIPPEDRELU: // min(alpha, max(0, x))
|
|
f([=](double x) { return (std::min(alpha, std::max(double(0.), x))); });
|
|
break;
|
|
case miopenActivationLEAKYRELU: // alpha * x | x<=0; x | x>0
|
|
f([=](double x) { return ((x > 0.) ? x : x * alpha); });
|
|
break;
|
|
case miopenActivationELU: // alpha * (exp(x)-1) | x<=0; x | x>0
|
|
f([=](double x) { return ((x > 0.) ? x : alpha * std::expm1(x)); });
|
|
break;
|
|
case miopenActivationCLAMP: // max(alpha, min(beta, x))
|
|
f([=](double x) { return (std::max(alpha, std::min(beta, x))); });
|
|
break;
|
|
// default: printf("ERROR: unknown neuron type: %d\n", activMode); break;
|
|
}
|
|
}
|
|
|
|
template <class T>
|
|
inline void activationHostInfer(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const std::vector<T> input,
|
|
std::vector<T>& output)
|
|
{
|
|
visitActivationHostInfer(activMode, gamma, beta, alpha, [&](auto f) {
|
|
miopen::par_for(input.size(), 1, [&](int index) {
|
|
output[index] = static_cast<T>(f(static_cast<double>(input[index])));
|
|
});
|
|
});
|
|
}
|
|
|
|
template <class F>
|
|
void visitActivationHostBwd(
|
|
miopenActivationMode_t activMode, double gamma, double beta, double alpha, F f)
|
|
{
|
|
switch(activMode)
|
|
{
|
|
case miopenActivationPASTHRU: // x
|
|
f([=](double dy, double, double) { return dy; });
|
|
break;
|
|
case miopenActivationLOGISTIC: // 1 / (1 + e^-x) //Sigmoid
|
|
f([=](double dy, double, double y) { return dy * y * (1 - y); });
|
|
break;
|
|
case miopenActivationTANH: // beta * tanh(alpha * x)
|
|
f([=](double dy, double, double y) { return dy * alpha * (beta - y * y / beta); });
|
|
break;
|
|
case miopenActivationRELU: // max(0, x)
|
|
f([=](double dy, double x, double) { return (x > 0) ? dy : 0; });
|
|
break;
|
|
case miopenActivationSOFTRELU: // log(1 + e^x) // bonomial normal log likelihood
|
|
f([=](double dy, double x, double) {
|
|
static const double threshold = 50.;
|
|
double expval = std::exp(std::min(x, threshold));
|
|
return dy * expval / (expval + 1.0);
|
|
});
|
|
break;
|
|
case miopenActivationABS: // abs(x)
|
|
f([=](double dy, double x, double) { return dy * ((x > 0) ? 1 : -1); });
|
|
break;
|
|
case miopenActivationPOWER: // (alpha + beta * x) ^ gamma
|
|
f([=](double, double x, double y) {
|
|
auto v = alpha + beta * x;
|
|
return v <= std::numeric_limits<double>::epsilon() ? 0 : gamma * beta * y / v;
|
|
});
|
|
break;
|
|
case miopenActivationCLIPPEDRELU: // min(alpha, max(0, x))
|
|
f([=](double dy, double x, double) { return (x > 0 && x <= alpha) ? dy : 0; });
|
|
break;
|
|
case miopenActivationLEAKYRELU: // alpha * x | x<=0; x | x>0
|
|
f([=](double dy, double x, double) { return dy * ((x > 0) ? 1 : alpha); });
|
|
break;
|
|
case miopenActivationELU: // alpah * (exp(x)-1) | x<=0; x | x>0
|
|
f([=](double dy, double x, double y) { return dy * ((x > 0) ? 1 : y + alpha); });
|
|
break;
|
|
case miopenActivationCLAMP: // max(alpha, min(beta, x))
|
|
f([=](double dy, double x, double) { return (x > alpha && x <= beta) ? dy : 0; });
|
|
break;
|
|
// default: printf("ERROR: unknown neuron type: %d\n", activMode); break;
|
|
}
|
|
}
|
|
|
|
template <class T, class U, class V>
|
|
inline void activationHostBnormBwd(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const std::vector<U> dyinput,
|
|
const std::vector<V> xinput,
|
|
std::vector<T>& output)
|
|
{
|
|
double dummy;
|
|
visitActivationHostBwd(activMode, gamma, beta, alpha, [&](auto f) {
|
|
miopen::par_for(dyinput.size(), 1, [&](int index) {
|
|
output[index] = static_cast<T>(
|
|
f(static_cast<double>(dyinput[index]), static_cast<double>(xinput[index]), dummy));
|
|
});
|
|
});
|
|
}
|
|
|
|
template <class T>
|
|
inline void activationHostBwd(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const std::vector<T> dyinput,
|
|
const std::vector<T> xinput,
|
|
const std::vector<T> yinput,
|
|
std::vector<T>& output)
|
|
{
|
|
visitActivationHostBwd(activMode, gamma, beta, alpha, [&](auto f) {
|
|
miopen::par_for(dyinput.size(), 1, [&](int index) {
|
|
output[index] = static_cast<T>(f(static_cast<double>(dyinput[index]),
|
|
static_cast<double>(xinput[index]),
|
|
static_cast<double>(yinput[index])));
|
|
});
|
|
});
|
|
}
|
|
|
|
inline void activationHostBwdElement(miopenActivationMode_t activMode,
|
|
double gamma,
|
|
double beta,
|
|
double alpha,
|
|
const double dyinput,
|
|
const double xinput,
|
|
const double yinput,
|
|
double& output)
|
|
{
|
|
visitActivationHostBwd(activMode, gamma, beta, alpha, [&](auto f) {
|
|
output = static_cast<double>(f(dyinput, xinput, yinput));
|
|
});
|
|
}
|
|
|
|
template <class T>
|
|
tensor<T> get_output_tensor(const miopen::ConvolutionDescriptor& filter,
|
|
const tensor<T>& input,
|
|
const tensor<T>& weights)
|
|
{
|
|
return tensor<T>{filter.GetForwardOutputTensor(input.desc, weights.desc, miopen_type<T>{})};
|
|
}
|