derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
3298 lines
136 KiB
C++
3298 lines
136 KiB
C++
// Copyright © Advanced Micro Devices, Inc., or its affiliates.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
#include <gtest/gtest.h>
|
|
#include <miopen/rnn.hpp>
|
|
#include <miopen/miopen.h>
|
|
#include "tensor_holder.hpp"
|
|
|
|
#include "compare_helper.hpp"
|
|
#include "gtest_desc_guard.hpp"
|
|
#include "gtest_handle_guard.hpp"
|
|
#include "../dropout_util.hpp"
|
|
#include "../rnn_util.hpp"
|
|
#include "../workspace.hpp"
|
|
|
|
#include <math.h>
|
|
namespace {
|
|
/// Specific version for compating CPU and GPU results, because this test is order-dependent
|
|
template <class VerifyT>
|
|
auto GruTestCompareResults(VerifyT&& verifier, double tolerance = 80.f)
|
|
-> std::pair<decltype(verifier.cpu()), decltype(verifier.gpu())>
|
|
{
|
|
const auto gpu_result = verifier.gpu();
|
|
const auto cpu_result = verifier.cpu();
|
|
if(!test_helpers::Compare(cpu_result, gpu_result, tolerance))
|
|
{
|
|
verifier.fail();
|
|
}
|
|
|
|
return std::make_pair(cpu_result, gpu_result);
|
|
}
|
|
|
|
/**********************************************
|
|
* CPU verification functions
|
|
*
|
|
**********************************************/
|
|
template <typename T>
|
|
void GRUFwdCPUVerify(const miopen::Handle& handle,
|
|
bool use_dropout,
|
|
miopen::DropoutDescriptor& dropoutDesc,
|
|
std::vector<T>& in,
|
|
std::vector<T>& wei, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans input1_trans
|
|
// hidden1_trans ... output_weight;
|
|
// bidirectional reversed weights ]
|
|
std::vector<T>& hy, // current/final hidden state
|
|
std::vector<T>& hx, // initial hidden state
|
|
std::vector<T>& out,
|
|
const std::vector<int>& in_n, // input batch size
|
|
int in_h, // input data length
|
|
int seqLength, // Number of iterations to unroll over
|
|
bool bidirection, // whether using bidirectional net
|
|
bool biased, // whether using bias
|
|
int hy_d, // 1 by numlayer (number of stacks of hidden layers) for
|
|
// unidirection, 2 by numlayer for bidirection
|
|
int hy_n, // equal to input batch size in_n[0]
|
|
int hy_h, // hidden state number
|
|
int out_h, // 1 by hy_h related function for unidirection, 2 by hy_h
|
|
// related function for bidirection
|
|
int inputMode,
|
|
std::vector<T>& rsvspace,
|
|
bool hx_is_null = false)
|
|
{
|
|
int batch_n = sumvc(in_n);
|
|
|
|
int numlayer = bidirection ? hy_d / 2 : hy_d;
|
|
int bi = bidirection ? 2 : 1;
|
|
|
|
int in_stride = in_h;
|
|
int out_stride = out_h;
|
|
int wei_stride = bi * 3 * hy_h;
|
|
int hy_stride = bi * 4 * hy_h;
|
|
int h_stride = bi * hy_h;
|
|
int uni_stride = hy_h;
|
|
int bi_stride = hy_h * bi;
|
|
|
|
if(inputMode == 1)
|
|
{
|
|
if(in_h != hy_h)
|
|
{
|
|
std::cout
|
|
<< "Verification cannot be completed: The input tensor size must equal to the "
|
|
<< "hidden state size of the network in SKIP_INPUT mode!" << std::endl;
|
|
return;
|
|
}
|
|
in_h = 0;
|
|
}
|
|
|
|
int wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride;
|
|
|
|
// initial dropoput
|
|
std::vector<rocrand_state_xorwow> dropout_states_host;
|
|
std::vector<unsigned char> dropout_reservespace_host;
|
|
std::vector<T> dropout_hid_state;
|
|
TensorDescGuard dropout_inputTensor;
|
|
TensorDescGuard dropout_outputTensor;
|
|
if(use_dropout)
|
|
{
|
|
size_t states_size = dropoutDesc.stateSizeInBytes / sizeof(rocrand_state_xorwow);
|
|
dropout_states_host = std::vector<rocrand_state_xorwow>(states_size);
|
|
InitKernelStateEmulator(dropout_states_host, dropoutDesc);
|
|
|
|
std::array<int, 2> drop_in_len = {{batch_n, hy_h * bi}};
|
|
std::array<int, 2> drop_in_str = {{hy_stride, 1}};
|
|
std::array<int, 2> drop_out_str = {{hy_h * bi, 1}};
|
|
miopenSetTensorDescriptor(
|
|
dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data());
|
|
miopenSetTensorDescriptor(
|
|
dropout_outputTensor, miopenFloat, 2, drop_in_len.data(), drop_out_str.data());
|
|
|
|
size_t reserveSpaceSizeInBytes = 0;
|
|
miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes);
|
|
size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char);
|
|
dropout_reservespace_host = std::vector<unsigned char>(reserve_size * (numlayer - 1),
|
|
static_cast<unsigned char>(1));
|
|
|
|
dropout_hid_state = std::vector<T>((numlayer - 1) * batch_n * hy_h * bi, static_cast<T>(0));
|
|
}
|
|
|
|
// forward emulator
|
|
for(int li = 0; li < numlayer; li++)
|
|
{
|
|
int hid_shift = li * batch_n * hy_stride;
|
|
int hx_shift = li * in_n.at(0) * h_stride;
|
|
int wei_shift_bias_temp = wei_shift_bias + li * 2 * wei_stride;
|
|
|
|
// from input
|
|
if(li == 0)
|
|
{
|
|
if(inputMode == 1)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 3; gi++)
|
|
{
|
|
rsvspace[hid_shift + bs * hy_stride + gi * hy_h + h] +=
|
|
in[bs * in_stride + h];
|
|
if(bidirection)
|
|
{
|
|
rsvspace[hid_shift + bs * hy_stride + (gi + 3) * hy_h + h] +=
|
|
in[bs * in_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// from bias
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < wei_stride; h++)
|
|
{
|
|
rsvspace[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
gemm_cpu(in.data(),
|
|
in_h,
|
|
batch_n,
|
|
in_stride,
|
|
false,
|
|
wei.data(), // wei_state.data(),
|
|
in_h,
|
|
hy_h * bi * 3,
|
|
in_stride,
|
|
true,
|
|
&rsvspace[hid_shift],
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
// from bias
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < wei_stride; h++)
|
|
{
|
|
rsvspace[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
int wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride;
|
|
int prelayer_shift = (li - 1) * batch_n * hy_stride + bi * 3 * hy_h;
|
|
if(use_dropout)
|
|
{
|
|
auto dropout_states_tmp = dropout_states_host;
|
|
size_t drop_out_offset = (static_cast<size_t>(li) - 1) * batch_n * hy_h * bi;
|
|
|
|
DropoutForwardVerify<T>(handle,
|
|
dropoutDesc,
|
|
miopen::deref(dropout_inputTensor.get()),
|
|
rsvspace,
|
|
miopen::deref(dropout_outputTensor.get()),
|
|
dropout_hid_state,
|
|
dropout_reservespace_host,
|
|
dropout_states_tmp,
|
|
prelayer_shift,
|
|
drop_out_offset,
|
|
drop_out_offset);
|
|
|
|
prelayer_shift = drop_out_offset;
|
|
}
|
|
|
|
gemm_cpu(use_dropout ? &dropout_hid_state[prelayer_shift] : &rsvspace[prelayer_shift],
|
|
hy_h * bi,
|
|
batch_n,
|
|
use_dropout ? hy_h * bi : hy_stride,
|
|
false,
|
|
&wei[wei_shift], //&wei_state[wei_shift],
|
|
hy_h * bi,
|
|
hy_h * bi * 3,
|
|
bi_stride,
|
|
true,
|
|
&rsvspace[hid_shift],
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
// from bias
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < wei_stride; h++)
|
|
{
|
|
rsvspace[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias_temp + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// from hidden state
|
|
int bacc = 0;
|
|
int baccbi = batch_n;
|
|
for(int ti = 0; ti < seqLength; ti++)
|
|
{
|
|
baccbi -= in_n.at(seqLength - 1 - ti);
|
|
int wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride;
|
|
int pretime_shift;
|
|
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
gemm_cpu(&hx[hx_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + bacc * hy_stride],
|
|
hy_h * 2,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 2; gi++)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + gi * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + gi * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&hx[hx_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 2 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + bacc * hy_stride + bi * 3 * hy_h],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + 2 * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
gemm_cpu(&hx[hx_shift + hy_n * hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + baccbi * hy_stride + 3 * hy_h],
|
|
hy_h * 2,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 2; gi++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
(3 + gi) * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h +
|
|
h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&hx[hx_shift + hy_n * hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 5 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
gemm_cpu(&hy[hx_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + bacc * hy_stride],
|
|
hy_h * 2,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 2; gi++)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + gi * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + gi * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&hy[hx_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 2 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + bacc * hy_stride + bi * 3 * hy_h],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + 2 * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
|
|
if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti))
|
|
{
|
|
gemm_cpu(
|
|
&hx[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h],
|
|
hy_h,
|
|
(in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride +
|
|
3 * hy_h],
|
|
hy_h * 2,
|
|
(in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti);
|
|
bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 2; gi++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
(3 + gi) * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h +
|
|
h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(
|
|
&hx[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h],
|
|
hy_h,
|
|
(in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 5 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride +
|
|
bi * 3 * hy_h + hy_h],
|
|
hy_h,
|
|
(in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti);
|
|
bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&hy[hx_shift + hy_n * hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + baccbi * hy_stride + 3 * hy_h],
|
|
hy_h * 2,
|
|
in_n.at(seqLength - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 2; gi++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
(3 + gi) * hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&hy[hx_shift + hy_n * hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - ti),
|
|
uni_stride,
|
|
false,
|
|
&wei[wei_shift + 5 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
true,
|
|
&rsvspace[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h];
|
|
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] +=
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2) *
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h];
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] = 0;
|
|
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
((1 -
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2)) *
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride +
|
|
2 * hy_h + h],
|
|
1) +
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2) *
|
|
hx[hx_shift + bs * uni_stride + h]);
|
|
}
|
|
else
|
|
{
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
((1 -
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2)) *
|
|
activfunc(
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h],
|
|
1));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
|
|
pretime_shift = li * batch_n * hy_stride +
|
|
(bacc - in_n.at(ti - 1)) * hy_stride + bi * 3 * hy_h;
|
|
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
((1 - activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2)) *
|
|
activfunc(
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h],
|
|
1) +
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2) *
|
|
rsvspace[pretime_shift + bs * hy_stride + h]);
|
|
}
|
|
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2);
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2);
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], 1);
|
|
|
|
// Update final state
|
|
hy[hx_shift + bs * uni_stride + h] =
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h];
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
pretime_shift = li * batch_n * hy_stride +
|
|
(baccbi + in_n.at(seqLength - 1 - ti)) * hy_stride + bi * 3 * hy_h +
|
|
hy_h;
|
|
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h];
|
|
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] +=
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], 2) *
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h];
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + h] =
|
|
0;
|
|
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
((1 - activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2)) *
|
|
activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
5 * hy_h + h],
|
|
1) +
|
|
activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2) *
|
|
hx[hx_shift + bs * uni_stride + hy_n * hy_h + h]);
|
|
}
|
|
else
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
((1 - activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2)) *
|
|
activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
5 * hy_h + h],
|
|
1));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti))
|
|
{
|
|
if(bs >= in_n.at(seqLength - ti))
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
(activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2) *
|
|
hx[hx_shift + bs * uni_stride + hy_n * hy_h + h]);
|
|
}
|
|
}
|
|
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h] +=
|
|
((1 - activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2)) *
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h],
|
|
1));
|
|
|
|
if(bs < in_n.at(seqLength - ti))
|
|
{
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
(activfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2) *
|
|
rsvspace[pretime_shift + bs * hy_stride + h]);
|
|
}
|
|
}
|
|
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h], 2);
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], 2);
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h], 1);
|
|
|
|
// Update final hidden state
|
|
hy[hx_shift + bs * uni_stride + hy_n * hy_h + h] =
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h];
|
|
}
|
|
}
|
|
}
|
|
|
|
bacc += in_n.at(ti);
|
|
}
|
|
}
|
|
|
|
// output
|
|
int prelayer_shift = (numlayer - 1) * batch_n * hy_stride + bi * 3 * hy_h;
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < out_h; h++)
|
|
{
|
|
out[bs * out_stride + h] = rsvspace[prelayer_shift + bs * hy_stride + h];
|
|
}
|
|
}
|
|
|
|
if(use_dropout)
|
|
{
|
|
for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++)
|
|
{
|
|
rsvspace.at(numlayer * batch_n * hy_stride * 2 + i) = dropout_hid_state.at(i);
|
|
}
|
|
auto p_drop_rsv = reinterpret_cast<unsigned char*>(&rsvspace.at(
|
|
numlayer * batch_n * hy_stride * 2 + (numlayer - 1) * batch_n * hy_h * bi));
|
|
for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++)
|
|
{
|
|
*(p_drop_rsv + i) = dropout_reservespace_host.at(i);
|
|
}
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
void GRUBwdDataCPUVerify(bool use_dropout,
|
|
miopen::DropoutDescriptor& dropoutDesc,
|
|
std::vector<T>& din,
|
|
std::vector<T>& wei, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans input1_trans
|
|
// hidden1_trans ... output_weight;
|
|
// bidirectional reversed weights ]
|
|
std::vector<T>& dhy, // current/final hidden state
|
|
std::vector<T>& dhx,
|
|
std::vector<T>& hx, // initial hidden state
|
|
std::vector<T>& out,
|
|
std::vector<T>& dout,
|
|
const std::vector<int>& in_n, // input batch size
|
|
int in_h, // input data length
|
|
int seqLength, // Number of iterations to unroll over
|
|
bool bidirection, // whether using bidirectional net
|
|
bool, // whether using bias
|
|
int hy_d, // 1 by numlayer (number of stacks of hidden layers)
|
|
// for unidirection, 2 by numlayer for bidirection
|
|
int hy_n, // equal to input batch size in_n[0]
|
|
int hy_h, // hidden state number
|
|
int out_h, // 1 by hy_h related function for unidirection, 2 by
|
|
// hy_h related function for bidirection
|
|
int inputMode,
|
|
std::vector<T>& rsvspace,
|
|
std::vector<T>& wkspace,
|
|
bool hx_is_null = false,
|
|
bool dhy_is_null = false)
|
|
{
|
|
int batch_n = sumvc(in_n);
|
|
(void)out;
|
|
|
|
int numlayer = bidirection ? hy_d / 2 : hy_d;
|
|
int bi = bidirection ? 2 : 1;
|
|
|
|
int in_stride = in_h;
|
|
int out_stride = out_h;
|
|
int wei_stride = bi * 3 * hy_h;
|
|
int hy_stride = bi * 4 * hy_h;
|
|
int h_stride = bi * hy_h;
|
|
int uni_stride = hy_h;
|
|
int bi_stride = hy_h * bi;
|
|
|
|
// initial hidden states
|
|
auto ihs = hy_d * hy_n * hy_h;
|
|
std::vector<T> dcx(ihs);
|
|
|
|
if(inputMode == 1)
|
|
{
|
|
if(in_h != hy_h)
|
|
{
|
|
std::cout
|
|
<< "Verification cannot be completed: The input tensor size must equal to the "
|
|
<< "hidden state size of the network in SKIP_INPUT mode!" << std::endl;
|
|
return;
|
|
}
|
|
in_h = 0;
|
|
}
|
|
|
|
// initial dropoput
|
|
TensorDescGuard dropout_inputTensor;
|
|
std::vector<unsigned char> dropout_reservespace_host;
|
|
if(use_dropout)
|
|
{
|
|
std::array<int, 2> drop_in_len = {{batch_n, hy_h * bi}};
|
|
std::array<int, 2> drop_in_str = {{hy_stride, 1}};
|
|
miopenSetTensorDescriptor(
|
|
dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data());
|
|
|
|
size_t reserveSpaceSizeInBytes = 0;
|
|
miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes);
|
|
size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char);
|
|
dropout_reservespace_host = std::vector<unsigned char>(reserve_size * (numlayer - 1),
|
|
static_cast<unsigned char>(0));
|
|
|
|
auto p_drop_rsv = reinterpret_cast<unsigned char*>(&rsvspace.at(
|
|
numlayer * batch_n * hy_stride * 2 + (numlayer - 1) * batch_n * hy_h * bi));
|
|
for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++)
|
|
{
|
|
dropout_reservespace_host.at(i) = *(p_drop_rsv + i);
|
|
}
|
|
}
|
|
|
|
// bwd data emulator
|
|
for(int li = numlayer - 1; li >= 0; li--)
|
|
{
|
|
int wei_shift = (in_h + hy_h) * wei_stride + li * (bi * hy_h + hy_h) * wei_stride;
|
|
int hid_shift = li * batch_n * hy_stride;
|
|
int hx_shift = li * in_n.at(0) * h_stride;
|
|
int weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride;
|
|
|
|
if(li == numlayer - 1)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < out_h; h++)
|
|
{
|
|
wkspace[hid_shift + bi * 3 * hy_h + bs * hy_stride + h] +=
|
|
dout[bs * out_stride + h];
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
int prelayer_shift = (li + 1) * batch_n * hy_stride;
|
|
|
|
gemm_cpu(&wkspace[prelayer_shift],
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
false,
|
|
&wei[wei_shift],
|
|
hy_h * bi,
|
|
hy_h * bi * 3,
|
|
bi_stride,
|
|
false,
|
|
&wkspace[hid_shift + bi * 3 * hy_h],
|
|
hy_h * bi,
|
|
batch_n,
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
if(use_dropout)
|
|
{
|
|
DropoutBackwardVerify<T>(dropoutDesc,
|
|
miopen::deref(dropout_inputTensor.get()),
|
|
wkspace,
|
|
miopen::deref(dropout_inputTensor.get()),
|
|
wkspace,
|
|
dropout_reservespace_host,
|
|
hid_shift + bi * 3 * hy_h,
|
|
hid_shift + bi * 3 * hy_h,
|
|
li * batch_n * hy_h * bi);
|
|
}
|
|
}
|
|
|
|
// from hidden state
|
|
int bacc = batch_n;
|
|
int baccbi = 0;
|
|
for(int ti = seqLength - 1; ti >= 0; ti--)
|
|
{
|
|
bacc -= in_n.at(ti);
|
|
|
|
if(ti == seqLength - 1)
|
|
{
|
|
if(!dhy_is_null)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
dhy[hx_shift + bs * uni_stride + h];
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] +=
|
|
dhy[hx_shift + bs * uni_stride + hy_n * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!dhy_is_null && in_n.at(ti) > in_n.at(ti + 1))
|
|
{
|
|
for(int bs = in_n.at(ti + 1); bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
dhy[hx_shift + bs * uni_stride + h];
|
|
}
|
|
}
|
|
}
|
|
|
|
int pretime_shift = li * batch_n * hy_stride + (bacc + in_n.at(ti)) * hy_stride;
|
|
|
|
gemm_cpu(&wkspace[pretime_shift],
|
|
hy_h * 2,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
false,
|
|
&wkspace[hid_shift + bacc * hy_stride + bi * 3 * hy_h],
|
|
hy_h,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = 0; bs < in_n.at(ti + 1); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] +=
|
|
wkspace[pretime_shift + bs * hy_stride + bi * 3 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + h], 2);
|
|
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] =
|
|
wkspace[pretime_shift + bs * hy_stride + 2 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + hy_h + h], 2);
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&wkspace[hid_shift + bacc * hy_stride + 2 * hy_h],
|
|
hy_h,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift + 2 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
false,
|
|
&wkspace[hid_shift + bacc * hy_stride + bi * 3 * hy_h],
|
|
hy_h,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = 0; bs < in_n.at(ti + 1); bs++)
|
|
{
|
|
auto subidx = hid_shift + (bacc + bs) * hy_stride + 2 * hy_h;
|
|
std::fill(wkspace.begin() + subidx, wkspace.begin() + subidx + hy_h, 0);
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
pretime_shift = li * batch_n * hy_stride +
|
|
(baccbi - in_n.at(seqLength - 2 - ti)) * hy_stride + hy_h * 3;
|
|
|
|
gemm_cpu(&wkspace[pretime_shift],
|
|
hy_h * 2,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift + hy_h * 3 * uni_stride],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
false,
|
|
&wkspace[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h] +=
|
|
wkspace[pretime_shift + bs * hy_stride + 3 * hy_h + hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + h], 2);
|
|
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] =
|
|
wkspace[pretime_shift + bs * hy_stride + 2 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + hy_h + h], 2);
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&wkspace[hid_shift + baccbi * hy_stride + 5 * hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift + 5 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
false,
|
|
&wkspace[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h],
|
|
hy_h,
|
|
in_n.at(seqLength - 1 - ti),
|
|
hy_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
auto subidx = hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h;
|
|
std::fill(wkspace.begin() + subidx, wkspace.begin() + (subidx + hy_h), 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] +=
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] *
|
|
(1 - activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2)) *
|
|
dervactivfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h],
|
|
1);
|
|
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h] =
|
|
(rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] *
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] *
|
|
dervactivfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h],
|
|
2));
|
|
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + h] +=
|
|
(wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] *
|
|
hx[hx_shift + bs * uni_stride + h] *
|
|
dervactivfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h],
|
|
2));
|
|
}
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + h] -=
|
|
(wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] *
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h],
|
|
1) *
|
|
dervactivfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2));
|
|
}
|
|
else
|
|
{
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + h] +=
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] *
|
|
(rsvspace[hid_shift + (bacc - in_n.at(ti - 1) + bs) * hy_stride +
|
|
bi * 3 * hy_h + h] -
|
|
activfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h],
|
|
1)) *
|
|
dervactivfunc(rsvspace[hid_shift + (bacc + bs) * hy_stride + h], 2);
|
|
}
|
|
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
wkspace[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] *
|
|
rsvspace[hid_shift + (bacc + bs) * hy_stride + hy_h + h +
|
|
numlayer * batch_n * hy_stride];
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] +=
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h] *
|
|
(1 - activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h],
|
|
2)) *
|
|
dervactivfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h], 1);
|
|
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h] =
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h +
|
|
h + numlayer * batch_n * hy_stride];
|
|
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h] *=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] *
|
|
dervactivfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h],
|
|
2));
|
|
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] +=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
hx[hx_shift + bs * uni_stride + hy_n * hy_h + h] *
|
|
dervactivfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2));
|
|
}
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] -=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h],
|
|
1) *
|
|
dervactivfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h],
|
|
2));
|
|
}
|
|
else
|
|
{
|
|
if(!hx_is_null &&
|
|
in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti) &&
|
|
bs >= in_n.at(seqLength - ti))
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] +=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
hx[hx_shift + bs * uni_stride + hy_n * hy_h + h] *
|
|
dervactivfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2));
|
|
}
|
|
|
|
if(bs < in_n.at(seqLength - ti))
|
|
{
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] +=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
rsvspace[hid_shift +
|
|
(baccbi + in_n.at(seqLength - 1 - ti) + bs) *
|
|
hy_stride +
|
|
bi * 3 * hy_h + hy_h + h] *
|
|
dervactivfunc(rsvspace[hid_shift + (baccbi + bs) * hy_stride +
|
|
3 * hy_h + h],
|
|
2));
|
|
}
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] -=
|
|
(wkspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
activfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h],
|
|
1) *
|
|
dervactivfunc(
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h],
|
|
2));
|
|
}
|
|
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + h +
|
|
numlayer * batch_n * hy_stride] =
|
|
wkspace[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] *
|
|
rsvspace[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h +
|
|
numlayer * batch_n * hy_stride];
|
|
}
|
|
}
|
|
}
|
|
|
|
baccbi += in_n.at(seqLength - 1 - ti);
|
|
}
|
|
|
|
// dhx
|
|
int pretime_shift = li * batch_n * hy_stride;
|
|
|
|
gemm_cpu(&wkspace[pretime_shift],
|
|
hy_h * 2,
|
|
in_n.at(0),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
false,
|
|
&dhx[hx_shift],
|
|
hy_h,
|
|
in_n.at(0),
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = 0; bs < in_n.at(0); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
dhx[hx_shift + bs * uni_stride + h] +=
|
|
wkspace[pretime_shift + bs * hy_stride + bi * 3 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + h], 2);
|
|
|
|
dcx[hx_shift + bs * uni_stride + h] =
|
|
wkspace[pretime_shift + bs * hy_stride + 2 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + bs * hy_stride + hy_h + h], 2);
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&dcx[hx_shift],
|
|
hy_h,
|
|
in_n.at(0),
|
|
uni_stride,
|
|
false,
|
|
&wei[weitime_shift + 2 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
false,
|
|
&dhx[hx_shift],
|
|
hy_h,
|
|
in_n.at(0),
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(bidirection)
|
|
{
|
|
int ti = seqLength - 1, cur_bat = 0, pre_bat = batch_n;
|
|
|
|
while(ti >= 0)
|
|
{
|
|
pre_bat -= in_n.at(ti);
|
|
if(in_n.at(ti) > cur_bat)
|
|
{
|
|
pretime_shift = li * batch_n * hy_stride + (pre_bat + cur_bat) * hy_stride;
|
|
|
|
gemm_cpu(&wkspace[pretime_shift + 3 * hy_h],
|
|
hy_h * 2,
|
|
(in_n.at(ti) - cur_bat),
|
|
hy_stride,
|
|
false,
|
|
&wei[weitime_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 2,
|
|
uni_stride,
|
|
false,
|
|
&dhx[hx_shift + hy_n * hy_h + cur_bat * hy_h],
|
|
hy_h,
|
|
(in_n.at(ti) - cur_bat),
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
for(int bs = cur_bat; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
dhx[hx_shift + bs * uni_stride + hy_n * hy_h + h] +=
|
|
wkspace[pretime_shift + (bs - cur_bat) * hy_stride + bi * 3 * hy_h +
|
|
hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + (bs - cur_bat) * hy_stride +
|
|
3 * hy_h + h],
|
|
2);
|
|
|
|
dcx[hx_shift + bs * uni_stride + hy_n * hy_h + h] =
|
|
wkspace[pretime_shift + (bs - cur_bat) * hy_stride + 5 * hy_h + h] *
|
|
activfunc(rsvspace[pretime_shift + (bs - cur_bat) * hy_stride +
|
|
4 * hy_h + h],
|
|
2);
|
|
}
|
|
}
|
|
|
|
gemm_cpu(&dcx[hx_shift + hy_n * hy_h + cur_bat * hy_h],
|
|
hy_h,
|
|
(in_n.at(ti) - cur_bat),
|
|
uni_stride,
|
|
false,
|
|
&wei[weitime_shift + 5 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h,
|
|
uni_stride,
|
|
false,
|
|
&dhx[hx_shift + hy_n * hy_h + cur_bat * hy_h],
|
|
hy_h,
|
|
(in_n.at(ti) - cur_bat),
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
}
|
|
cur_bat = in_n.at(ti--);
|
|
}
|
|
}
|
|
}
|
|
|
|
// dinput
|
|
if(inputMode == 1)
|
|
{
|
|
for(int bs = 0; bs < batch_n; bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
for(int gi = 0; gi < 3; gi++)
|
|
{
|
|
din[bs * in_stride + h] += wkspace[bs * hy_stride + gi * hy_h + h];
|
|
if(bidirection)
|
|
{
|
|
din[bs * in_stride + h] += wkspace[bs * hy_stride + (gi + 3) * hy_h + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
gemm_cpu(wkspace.data(),
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
false,
|
|
wei.data(),
|
|
in_h,
|
|
hy_h * bi * 3,
|
|
in_stride,
|
|
false,
|
|
din.data(),
|
|
in_h,
|
|
batch_n,
|
|
in_stride,
|
|
1,
|
|
1);
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
void GRUBwdWeightCPUVerify(bool use_dropout,
|
|
std::vector<T>& in,
|
|
std::vector<T>& dwei, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans
|
|
// input1_trans hidden1_trans ...
|
|
// output_weight; bidirectional
|
|
// reversed weights ]
|
|
std::vector<T>& hx, // initial hidden state
|
|
const std::vector<int>& in_n, // input batch size
|
|
int in_h, // input data length
|
|
int seqLength, // Number of iterations to unroll over
|
|
bool bidirection, // whether using bidirectional net
|
|
bool biased, // whether using bias
|
|
int hy_d, // 1 by numlayer (number of stacks of hidden
|
|
// layers) for unidirection, 2 by numlayer for
|
|
// bidirection
|
|
int hy_n, // equal to input batch size in_n[0]
|
|
int hy_h, // hidden state number
|
|
// by hy_h related function for bidirection
|
|
int inputMode,
|
|
std::vector<T>& rsvspace,
|
|
std::vector<T>& wkspace,
|
|
bool hx_is_null = false)
|
|
{
|
|
int batch_n = sumvc(in_n);
|
|
int numlayer = bidirection ? hy_d / 2 : hy_d;
|
|
int bi = bidirection ? 2 : 1;
|
|
|
|
int in_stride = in_h;
|
|
int wei_stride = bi * 3 * hy_h;
|
|
int hy_stride = bi * 4 * hy_h;
|
|
int h_stride = bi * hy_h;
|
|
int uni_stride = hy_h;
|
|
int bi_stride = hy_h * bi;
|
|
|
|
if(inputMode == 1)
|
|
{
|
|
if(in_h != hy_h)
|
|
{
|
|
std::cout
|
|
<< "Verification cannot be completed: The input tensor size must equal to the "
|
|
<< "hidden state size of the network in SKIP_INPUT mode!" << std::endl;
|
|
return;
|
|
}
|
|
in_h = 0;
|
|
}
|
|
|
|
int wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride;
|
|
|
|
// bwd weights emulator
|
|
for(int li = 0; li < numlayer; li++)
|
|
{
|
|
// between layers
|
|
if(li == 0)
|
|
{
|
|
if(inputMode == 0)
|
|
{
|
|
gemm_cpu(wkspace.data(),
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
true,
|
|
in.data(),
|
|
in_h,
|
|
batch_n,
|
|
in_stride,
|
|
false,
|
|
dwei.data(),
|
|
in_h,
|
|
hy_h * bi * 3,
|
|
in_stride,
|
|
1,
|
|
1);
|
|
}
|
|
|
|
if(biased)
|
|
{
|
|
for(int h = 0; h < wei_stride; h++)
|
|
{
|
|
for(int w = 0; w < batch_n; w++)
|
|
{
|
|
dwei[wei_shift_bias + h] += wkspace[w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
int prelayer_shift =
|
|
use_dropout ? 2 * numlayer * batch_n * hy_stride + (li - 1) * batch_n * hy_h * bi
|
|
: (li - 1) * batch_n * hy_stride + bi * hy_h * 3;
|
|
int hid_shift = li * batch_n * hy_stride;
|
|
int wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride;
|
|
|
|
gemm_cpu(&wkspace[hid_shift],
|
|
hy_h * bi * 3,
|
|
batch_n,
|
|
hy_stride,
|
|
true,
|
|
&rsvspace[prelayer_shift],
|
|
hy_h * bi,
|
|
batch_n,
|
|
use_dropout ? hy_h * bi : hy_stride,
|
|
false,
|
|
&dwei[wei_shift],
|
|
hy_h * bi,
|
|
hy_h * bi * 3,
|
|
bi_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
wei_shift = wei_shift_bias + li * 2 * wei_stride;
|
|
|
|
for(int h = 0; h < wei_stride; h++)
|
|
{
|
|
for(int w = 0; w < batch_n; w++)
|
|
{
|
|
dwei[wei_shift + h] += wkspace[hid_shift + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// between time
|
|
int bacc = 0;
|
|
for(int ti = 0; ti < seqLength; ti++)
|
|
{
|
|
int hid_shift = li * batch_n * hy_stride + bacc * hy_stride;
|
|
int hx_shift = li * in_n.at(0) * h_stride;
|
|
int wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride;
|
|
int pretime_shift;
|
|
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + bs * hy_stride + 2 * hy_h + h] *=
|
|
activfunc(rsvspace[hid_shift + bs * hy_stride + hy_h + h], 2);
|
|
}
|
|
}
|
|
|
|
// between time
|
|
if(ti == 0)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
gemm_cpu(&wkspace[hid_shift],
|
|
hy_h * 3,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
true,
|
|
&hx[hx_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&dwei[wei_shift],
|
|
hy_h,
|
|
hy_h * 3,
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride;
|
|
|
|
for(int h = 0; h < hy_h * 3; h++)
|
|
{
|
|
for(int w = 0; w < in_n.at(ti); w++)
|
|
{
|
|
dwei[bias_shift + h] += wkspace[hid_shift + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
pretime_shift =
|
|
li * batch_n * hy_stride + (bacc - in_n.at(ti - 1)) * hy_stride + bi * 3 * hy_h;
|
|
|
|
gemm_cpu(&wkspace[hid_shift],
|
|
hy_h * 3,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
true,
|
|
&rsvspace[pretime_shift],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
false,
|
|
&dwei[wei_shift],
|
|
hy_h,
|
|
hy_h * 3,
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride;
|
|
|
|
for(int h = 0; h < hy_h * 3; h++)
|
|
{
|
|
for(int w = 0; w < in_n.at(ti); w++)
|
|
{
|
|
dwei[bias_shift + h] += wkspace[hid_shift + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(bidirection)
|
|
{
|
|
for(int bs = 0; bs < in_n.at(ti); bs++)
|
|
{
|
|
for(int h = 0; h < hy_h; h++)
|
|
{
|
|
wkspace[hid_shift + bs * hy_stride + 5 * hy_h + h] *=
|
|
activfunc(rsvspace[hid_shift + bs * hy_stride + 4 * hy_h + h], 2);
|
|
}
|
|
}
|
|
|
|
if(ti == seqLength - 1)
|
|
{
|
|
if(!hx_is_null)
|
|
{
|
|
gemm_cpu(&wkspace[hid_shift + 3 * hy_h],
|
|
hy_h * 3,
|
|
in_n.at(ti),
|
|
hy_stride,
|
|
true,
|
|
&hx[hx_shift + hy_n * hy_h],
|
|
hy_h,
|
|
in_n.at(ti),
|
|
uni_stride,
|
|
false,
|
|
&dwei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 3,
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride;
|
|
|
|
for(int h = 0; h < hy_h * 3; h++)
|
|
{
|
|
for(int w = 0; w < in_n.at(ti); w++)
|
|
{
|
|
dwei[bias_shift + 3 * hy_h + h] +=
|
|
wkspace[hid_shift + 3 * hy_h + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!hx_is_null && in_n.at(ti) > in_n.at(ti + 1))
|
|
{
|
|
gemm_cpu(&wkspace[hid_shift + 3 * hy_h + in_n.at(ti + 1) * hy_stride],
|
|
hy_h * 3,
|
|
(in_n.at(ti) - in_n.at(ti + 1)),
|
|
hy_stride,
|
|
true,
|
|
&hx[hx_shift + hy_n * hy_h + in_n.at(ti + 1) * hy_h],
|
|
hy_h,
|
|
(in_n.at(ti) - in_n.at(ti + 1)),
|
|
uni_stride,
|
|
false,
|
|
&dwei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 3,
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride;
|
|
|
|
for(int h = 0; h < hy_h * 3; h++)
|
|
{
|
|
for(int w = in_n.at(ti + 1); w < in_n.at(ti); w++)
|
|
{
|
|
dwei[bias_shift + 3 * hy_h + h] +=
|
|
wkspace[hid_shift + 3 * hy_h + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pretime_shift =
|
|
li * batch_n * hy_stride + (bacc + in_n.at(ti)) * hy_stride + bi * 3 * hy_h;
|
|
|
|
gemm_cpu(&wkspace[hid_shift + 3 * hy_h],
|
|
hy_h * 3,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
true,
|
|
&rsvspace[pretime_shift + hy_h],
|
|
hy_h,
|
|
in_n.at(ti + 1),
|
|
hy_stride,
|
|
false,
|
|
&dwei[wei_shift + 3 * hy_h * uni_stride],
|
|
hy_h,
|
|
hy_h * 3,
|
|
uni_stride,
|
|
1,
|
|
1);
|
|
|
|
if(biased)
|
|
{
|
|
int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride;
|
|
|
|
for(int h = 0; h < hy_h * 3; h++)
|
|
{
|
|
for(int w = 0; w < in_n.at(ti + 1); w++)
|
|
{
|
|
dwei[bias_shift + 3 * hy_h + h] +=
|
|
wkspace[hid_shift + 3 * hy_h + w * hy_stride + h];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
bacc += in_n.at(ti);
|
|
}
|
|
}
|
|
}
|
|
|
|
//////=========END CPU VERIFICATION FUNCTIONS=============
|
|
|
|
inline std::vector<int> GenBatchSeq(const int batchSize, const int seqLength)
|
|
{
|
|
|
|
static constexpr int modval = 3;
|
|
|
|
int currentval = batchSize;
|
|
std::vector<int> batchSeq;
|
|
batchSeq.reserve(seqLength);
|
|
for(int i = 0; i < seqLength; i++)
|
|
{
|
|
if(i > 0)
|
|
{
|
|
int nvalue = currentval - prng::gen_0_to_B(modval);
|
|
currentval = (nvalue < 1) ? 1 : nvalue;
|
|
}
|
|
batchSeq.push_back(currentval);
|
|
}
|
|
return batchSeq;
|
|
}
|
|
|
|
//****************************************************
|
|
// FORWARD INFERENCE
|
|
//****************************************************
|
|
template <class T>
|
|
struct verify_forward_infer_gru
|
|
{
|
|
std::vector<T> input;
|
|
std::vector<T> initHidden;
|
|
std::vector<T> weights;
|
|
std::vector<int> batch_seq;
|
|
int hiddenSize;
|
|
int seqLength;
|
|
int nLayers;
|
|
int biasMode;
|
|
int dirMode;
|
|
int inputMode;
|
|
int batch_n;
|
|
int inputVecLen;
|
|
miopenRNNDescriptor_t rnnDesc;
|
|
size_t realHiddenSize;
|
|
bool nohx;
|
|
bool nohy;
|
|
|
|
verify_forward_infer_gru(miopenRNNDescriptor_t pRD,
|
|
const std::vector<T>& px,
|
|
const std::vector<T>& phx,
|
|
const std::vector<T>& pW,
|
|
const std::vector<int>& pBS,
|
|
const int pHS,
|
|
const int pBN,
|
|
const int pS,
|
|
const int pNL,
|
|
const int pBM,
|
|
const int pDM,
|
|
const int pIM,
|
|
const int pVL,
|
|
const size_t pHXZ,
|
|
const bool pnohx = false,
|
|
const bool pnohy = false)
|
|
: input(px),
|
|
initHidden(phx),
|
|
weights(pW),
|
|
batch_seq(pBS),
|
|
hiddenSize(pHS),
|
|
seqLength(pS),
|
|
nLayers(pNL),
|
|
biasMode(pBM),
|
|
dirMode(pDM),
|
|
inputMode(pIM),
|
|
batch_n(pBN),
|
|
inputVecLen(pVL),
|
|
rnnDesc(pRD),
|
|
realHiddenSize(pHXZ),
|
|
nohx(pnohx),
|
|
nohy(pnohy)
|
|
{
|
|
if(!nohx)
|
|
initHidden = phx; // this may be intentionally a nullptr
|
|
else
|
|
initHidden.resize(realHiddenSize);
|
|
}
|
|
|
|
std::tuple<std::vector<T>> cpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
int bi = dirMode != 0 ? 2 : 1;
|
|
int hy_h = hiddenSize;
|
|
int bi_stride = bi * hy_h;
|
|
size_t out_sz = 0;
|
|
|
|
size_t reserveSpaceSize;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, outputDescs.data(), &out_sz);
|
|
miopenGetRNNTrainingReserveSize(
|
|
&handle, rnnDesc, seqLength, inputDescs.data(), &reserveSpaceSize);
|
|
std::vector<T> reserveSpace(reserveSpaceSize / sizeof(T));
|
|
std::vector<T> output(out_sz / sizeof(T));
|
|
std::vector<T> hiddenState(initHidden.size());
|
|
|
|
GRUFwdCPUVerify(handle,
|
|
false,
|
|
miopen::deref(miopen::deref(rnnDesc).dropoutDesc),
|
|
input,
|
|
weights, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans input1_trans
|
|
// hidden1_trans ... output_weight;
|
|
// bidirectional reversed weights ]
|
|
hiddenState, // current/final hidden state
|
|
initHidden, // initial hidden state
|
|
output,
|
|
batch_seq, // input batch size
|
|
inputVecLen, // input data length
|
|
seqLength, // Number of iterations to unroll over
|
|
dirMode, // whether using bidirectional net
|
|
biasMode, // whether using bias
|
|
bi * nLayers, // 1 by numlayer (number of stacks of hidden layers) for
|
|
// unidirection, 2 by numlayer for bidirection
|
|
batch_seq.at(0), // equal to input batch size in_n[0]
|
|
hiddenSize, // hidden state number
|
|
bi_stride, // 1 by hy_h related function for unidirection, 2 by hy_h
|
|
// related function for bidirection
|
|
inputMode,
|
|
reserveSpace,
|
|
nohx);
|
|
return std::make_tuple(output);
|
|
}
|
|
|
|
std::tuple<std::vector<T>> gpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
size_t out_sz = 0;
|
|
size_t workspace_size = 0;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
miopenGetRNNWorkspaceSize(&handle, rnnDesc, seqLength, inputDescs.data(), &workspace_size);
|
|
Workspace wspace{workspace_size};
|
|
|
|
auto input_dev = handle.Write(input);
|
|
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, outputDescs.data(), &out_sz);
|
|
std::vector<T> output(out_sz / sizeof(T));
|
|
auto output_dev = handle.Write(output);
|
|
|
|
auto weights_dev = handle.Write(weights);
|
|
auto hy = initHidden;
|
|
std::fill(hy.begin(), hy.end(), 0.);
|
|
auto hy_dev = handle.Write(hy);
|
|
|
|
std::vector<int> hlens(3, 0);
|
|
hlens[0] = nLayers * (dirMode != 0 ? 2 : 1);
|
|
hlens[1] = batch_seq[0];
|
|
hlens[2] = hiddenSize;
|
|
miopen::TensorDescriptor hiddenDesc(miopen::deref(rnnDesc).dataType, hlens);
|
|
|
|
std::vector<int> wlen(1, 0);
|
|
wlen[0] = weights.size();
|
|
miopen::TensorDescriptor weightDesc(miopen::deref(rnnDesc).dataType, wlen);
|
|
auto hx_dev = nohx ? miopen::Allocator::ManageDataPtr{} : handle.Write(initHidden);
|
|
|
|
miopenRNNForwardInference(&handle,
|
|
rnnDesc,
|
|
seqLength,
|
|
inputDescs.data(),
|
|
input_dev.get(),
|
|
&hiddenDesc,
|
|
hx_dev.get(),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
&weightDesc,
|
|
weights_dev.get(),
|
|
outputDescs.data(),
|
|
output_dev.get(),
|
|
&hiddenDesc,
|
|
((nohy) ? nullptr : hy_dev.get()),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
wspace.ptr(),
|
|
wspace.size());
|
|
return std::make_tuple((handle.Read<T>(output_dev, output.size())));
|
|
}
|
|
|
|
void fail() const
|
|
{
|
|
std::stringstream ss{};
|
|
ss << "./bin/MIOpenDriver rnn -n ";
|
|
for(int i = 0; i < seqLength; i++)
|
|
{
|
|
if(i < seqLength - 1)
|
|
{
|
|
ss << batch_seq.at(i) << ",";
|
|
}
|
|
else
|
|
{
|
|
ss << batch_seq.at(i);
|
|
}
|
|
}
|
|
ss << " -m gru -k " << seqLength << " -H " << hiddenSize << " -W " << inputVecLen << " -l "
|
|
<< nLayers << " -F 0 -r " << dirMode << " -b " << biasMode << " -p " << inputMode
|
|
<< std::endl;
|
|
|
|
ss << "inputMode: " << inputMode << " biasMode: " << biasMode << " dirMode: " << dirMode
|
|
<< std::endl;
|
|
ss << "hz: " << hiddenSize << " batch_n: " << batch_n << " seqLength: " << seqLength
|
|
<< " inputLen: " << inputVecLen << " numLayers: " << nLayers << std::endl;
|
|
ss << "Forward Inference GRU: " << std::endl;
|
|
ss << "Output tensor output failed verification." << std::endl;
|
|
GTEST_FAIL() << ss.str();
|
|
}
|
|
};
|
|
//~~~~~~~~~~~~ END FWD INFERENCE ~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
//****************************************************
|
|
// FORWARD TRAIN
|
|
//****************************************************
|
|
template <class T>
|
|
struct verify_forward_train_gru
|
|
{
|
|
std::vector<T> input;
|
|
std::vector<T> initHidden;
|
|
std::vector<T> weights;
|
|
std::vector<int> batch_seq;
|
|
int hiddenSize;
|
|
int seqLength;
|
|
int nLayers;
|
|
int biasMode;
|
|
int dirMode;
|
|
int inputMode;
|
|
int batch_n;
|
|
int inputVecLen;
|
|
miopenRNNDescriptor_t rnnDesc;
|
|
size_t realHiddenSize;
|
|
bool nohx;
|
|
bool nohy;
|
|
bool use_dropout;
|
|
|
|
verify_forward_train_gru(miopenRNNDescriptor_t pRD,
|
|
const std::vector<T>& px,
|
|
const std::vector<T>& phx,
|
|
const std::vector<T>& pW,
|
|
const std::vector<int>& pBS,
|
|
const int pHS,
|
|
const int pBN,
|
|
const int pS,
|
|
const int pNL,
|
|
const int pBM,
|
|
const int pDM,
|
|
const int pIM,
|
|
const int pVL,
|
|
const size_t pHXZ,
|
|
const bool pnohx = false,
|
|
const bool pnohy = false,
|
|
const bool puse_dropout = false)
|
|
: input(px),
|
|
initHidden(phx),
|
|
weights(pW),
|
|
batch_seq(pBS),
|
|
hiddenSize(pHS),
|
|
seqLength(pS),
|
|
nLayers(pNL),
|
|
biasMode(pBM),
|
|
dirMode(pDM),
|
|
inputMode(pIM),
|
|
batch_n(pBN),
|
|
inputVecLen(pVL),
|
|
rnnDesc(pRD),
|
|
realHiddenSize(pHXZ),
|
|
nohx(pnohx),
|
|
nohy(pnohy),
|
|
use_dropout(puse_dropout)
|
|
{
|
|
if(!nohx)
|
|
initHidden = phx; // this may be intentionally a nullptr
|
|
else
|
|
initHidden.resize(realHiddenSize);
|
|
}
|
|
|
|
std::tuple<std::vector<T>, std::vector<T>, std::vector<T>> cpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
int bi = dirMode != 0 ? 2 : 1;
|
|
int hy_h = hiddenSize;
|
|
int bi_stride = bi * hy_h;
|
|
size_t out_sz = 0;
|
|
size_t reserveSpaceSize;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, outputDescs.data(), &out_sz);
|
|
miopenGetRNNTrainingReserveSize(
|
|
&handle, rnnDesc, seqLength, inputDescs.data(), &reserveSpaceSize);
|
|
std::vector<T> reserveSpace((reserveSpaceSize + sizeof(T) - 1) / sizeof(T));
|
|
std::vector<T> output(out_sz / sizeof(T));
|
|
std::vector<T> hiddenState(initHidden.size());
|
|
|
|
GRUFwdCPUVerify(handle,
|
|
use_dropout,
|
|
miopen::deref(miopen::deref(rnnDesc).dropoutDesc),
|
|
input,
|
|
weights, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans input1_trans
|
|
// hidden1_trans ... output_weight;
|
|
// bidirectional reversed weights ]
|
|
hiddenState, // current/final hidden state
|
|
initHidden, // initial hidden state
|
|
output,
|
|
batch_seq, // input batch size
|
|
inputVecLen, // input data length
|
|
seqLength, // Number of iterations to unroll over
|
|
dirMode, // whether using bidirectional net
|
|
biasMode, // whether using bias
|
|
bi * nLayers, // 1 by numlayer (number of stacks of hidden layers) for
|
|
// unidirection, 2 by numlayer for bidirection
|
|
batch_seq.at(0), // equal to input batch size in_n[0]
|
|
hiddenSize, // hidden state number
|
|
bi_stride, // 1 by hy_h related function for unidirection, 2 by hy_h
|
|
// related function for bidirection
|
|
inputMode,
|
|
reserveSpace,
|
|
nohx);
|
|
|
|
auto retSet = std::make_tuple(output, (nohy ? initHidden : hiddenState), reserveSpace);
|
|
return retSet;
|
|
}
|
|
|
|
std::tuple<std::vector<T>, std::vector<T>, std::vector<T>> gpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
size_t out_sz = 0;
|
|
size_t workspace_size = 0;
|
|
size_t reserveSpaceSize = 0;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
miopenGetRNNWorkspaceSize(&handle, rnnDesc, seqLength, inputDescs.data(), &workspace_size);
|
|
Workspace wspace{workspace_size};
|
|
|
|
miopenGetRNNTrainingReserveSize(
|
|
&handle, rnnDesc, seqLength, inputDescs.data(), &reserveSpaceSize);
|
|
reserveSpaceSize = (reserveSpaceSize + sizeof(T) - 1) & ~(sizeof(T) - 1);
|
|
assert(reserveSpaceSize % sizeof(T) == 0);
|
|
Workspace rspace{reserveSpaceSize};
|
|
|
|
auto input_dev = handle.Write(input);
|
|
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, outputDescs.data(), &out_sz);
|
|
std::vector<T> output(out_sz / sizeof(T));
|
|
auto output_dev = handle.Write(output);
|
|
|
|
auto weights_dev = handle.Write(weights);
|
|
|
|
auto hy = initHidden;
|
|
std::fill(hy.begin(), hy.end(), 0.);
|
|
auto hy_dev = handle.Write(hy);
|
|
|
|
std::vector<int> hlens(3, 0);
|
|
hlens[0] = nLayers * (dirMode != 0 ? 2 : 1);
|
|
hlens[1] = batch_seq[0];
|
|
hlens[2] = hiddenSize;
|
|
miopen::TensorDescriptor hiddenDesc(miopen::deref(rnnDesc).dataType, hlens);
|
|
|
|
std::vector<int> wlen(1, 0);
|
|
wlen[0] = weights.size();
|
|
miopen::TensorDescriptor weightDesc(miopen::deref(rnnDesc).dataType, wlen);
|
|
|
|
auto hx_dev = nohx ? miopen::Allocator::ManageDataPtr{} : handle.Write(initHidden);
|
|
|
|
miopenRNNForwardTraining(&handle,
|
|
rnnDesc,
|
|
seqLength,
|
|
inputDescs.data(),
|
|
input_dev.get(),
|
|
&hiddenDesc,
|
|
hx_dev.get(),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
&weightDesc,
|
|
weights_dev.get(),
|
|
outputDescs.data(),
|
|
output_dev.get(),
|
|
&hiddenDesc,
|
|
((nohy) ? nullptr : hy_dev.get()),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
wspace.ptr(),
|
|
wspace.size(),
|
|
rspace.ptr(),
|
|
rspace.size());
|
|
|
|
auto outdata = handle.Read<T>(output_dev, output.size());
|
|
|
|
auto retSet = std::make_tuple(outdata,
|
|
(nohy ? initHidden : handle.Read<T>(hy_dev, hy.size())),
|
|
rspace.Read<std::vector<T>>());
|
|
return retSet;
|
|
}
|
|
|
|
void fail() const
|
|
{
|
|
std::stringstream ss{};
|
|
ss << "./bin/MIOpenDriver rnn -n ";
|
|
for(int i = 0; i < seqLength; i++)
|
|
{
|
|
if(i < seqLength - 1)
|
|
{
|
|
ss << batch_seq.at(i) << ",";
|
|
}
|
|
else
|
|
{
|
|
ss << batch_seq.at(i);
|
|
}
|
|
}
|
|
ss << " -m gru -k " << seqLength << " -H " << hiddenSize << " -W " << inputVecLen << " -l "
|
|
<< nLayers << " -F 0 -r " << dirMode << " -b " << biasMode << " -p " << inputMode
|
|
<< std::endl;
|
|
|
|
ss << "inputMode: " << inputMode << " biasMode: " << biasMode << " dirMode: " << dirMode
|
|
<< std::endl;
|
|
ss << "hz: " << hiddenSize << " batch_n: " << batch_n << " seqLength: " << seqLength
|
|
<< " inputLen: " << inputVecLen << " numLayers: " << nLayers
|
|
<< " useDropout: " << int(use_dropout) << std::endl;
|
|
ss << "Forward Train GRU: " << std::endl;
|
|
GTEST_FAIL() << ss.str();
|
|
}
|
|
};
|
|
//~~~~~~~~~~~~ END FWD TRAIN ~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
//****************************************************
|
|
// BACKWARDS DATA
|
|
//****************************************************
|
|
template <class T>
|
|
struct verify_backward_data_gru
|
|
{
|
|
std::vector<T> yin; // Y
|
|
std::vector<T> dy; // dY
|
|
std::vector<T> dhy; // dHY
|
|
std::vector<T> initHidden; // HX
|
|
std::vector<T> weights;
|
|
std::vector<T> reserveSpace;
|
|
std::vector<int> batch_seq;
|
|
int hiddenSize;
|
|
int seqLength;
|
|
int nLayers;
|
|
int biasMode;
|
|
int dirMode;
|
|
int inputMode;
|
|
int batch_n;
|
|
int inputVecLen;
|
|
miopenRNNDescriptor_t rnnDesc;
|
|
size_t realHiddenSize;
|
|
bool nohx;
|
|
bool nodhy;
|
|
bool nodhx;
|
|
bool use_dropout;
|
|
|
|
verify_backward_data_gru(miopenRNNDescriptor_t pRD,
|
|
const std::vector<T>& py,
|
|
const std::vector<T>& pdy,
|
|
const std::vector<T>& pdhy,
|
|
const std::vector<T>& phx,
|
|
const std::vector<T>& pW,
|
|
const std::vector<T>& pRS,
|
|
const std::vector<int>& pBS,
|
|
const int pHS,
|
|
const int pBN,
|
|
const int pS,
|
|
const int pNL,
|
|
const int pBM,
|
|
const int pDM,
|
|
const int pIM,
|
|
const int pVL,
|
|
const size_t pHXZ,
|
|
const bool pnohx = false,
|
|
const bool pnodhy = false,
|
|
const bool pnodhx = false,
|
|
const bool puse_dropout = false)
|
|
: yin(py),
|
|
dy(pdy),
|
|
dhy(pdhy),
|
|
initHidden(phx),
|
|
weights(pW),
|
|
reserveSpace(pRS),
|
|
batch_seq(pBS),
|
|
hiddenSize(pHS),
|
|
seqLength(pS),
|
|
nLayers(pNL),
|
|
biasMode(pBM),
|
|
dirMode(pDM),
|
|
inputMode(pIM),
|
|
batch_n(pBN),
|
|
inputVecLen(pVL),
|
|
rnnDesc(pRD),
|
|
realHiddenSize(pHXZ),
|
|
nohx(pnohx),
|
|
nodhy(pnodhy),
|
|
nodhx(pnodhx),
|
|
use_dropout(puse_dropout)
|
|
{
|
|
if(!nohx)
|
|
initHidden = phx; // this may be intentionally a nullptr
|
|
else
|
|
initHidden.resize(realHiddenSize);
|
|
|
|
if(!nodhy)
|
|
dhy = pdhy; // this may be intentionally a nullptr
|
|
else
|
|
dhy.resize(realHiddenSize);
|
|
}
|
|
|
|
std::tuple<std::vector<T>, std::vector<T>, std::vector<T>, std::vector<T>> cpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
int bi = dirMode != 0 ? 2 : 1;
|
|
int hy_h = hiddenSize;
|
|
int bi_stride = bi * hy_h;
|
|
size_t workspace_size;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
// Outputs ----------
|
|
size_t in_sz = 0;
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, inputDescs.data(), &in_sz);
|
|
miopenGetRNNWorkspaceSize(&handle, rnnDesc, seqLength, inputDescs.data(), &workspace_size);
|
|
std::vector<T> workSpace(workspace_size / sizeof(T));
|
|
std::vector<T> dx(in_sz / sizeof(T));
|
|
std::vector<T> dhx(initHidden.size());
|
|
|
|
GRUBwdDataCPUVerify(use_dropout,
|
|
miopen::deref(miopen::deref(rnnDesc).dropoutDesc),
|
|
dx, // DX (output)
|
|
weights, // [ input_state_weight_trans
|
|
// hidden_state_weight0_trans input1_trans
|
|
// hidden1_trans ... output_weight;
|
|
// bidirectional reversed weights ]
|
|
dhy, // current/final hidden state
|
|
dhx, // DHX (output)
|
|
initHidden, // HX initial hidden state
|
|
yin, // Y
|
|
dy, // DY
|
|
batch_seq, // input batch size
|
|
inputVecLen, // input data length
|
|
seqLength, // Number of iterations to unroll over
|
|
dirMode, // whether using bidirectional net
|
|
biasMode, // whether using bias
|
|
bi * nLayers, // 1 by numlayer (number of stacks of hidden layers)
|
|
// for unidirection, 2 by numlayer for bidirection
|
|
batch_seq.at(0), // equal to input batch size in_n[0]
|
|
hiddenSize, // hidden state number
|
|
bi_stride, // 1 by hy_h related function for unidirection, 2 by
|
|
// hy_h related function for bidirection
|
|
inputMode,
|
|
reserveSpace,
|
|
workSpace,
|
|
nohx,
|
|
nodhy);
|
|
return std::make_tuple(dx, (nodhx ? initHidden : dhx), reserveSpace, workSpace);
|
|
}
|
|
|
|
std::tuple<std::vector<T>, std::vector<T>, std::vector<T>, std::vector<T>> gpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
size_t out_sz = 0;
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
size_t workspace_size = 0;
|
|
miopenGetRNNWorkspaceSize(&handle, rnnDesc, seqLength, inputDescs.data(), &workspace_size);
|
|
Workspace wspace{workspace_size};
|
|
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, outputDescs.data(), &out_sz);
|
|
auto yin_dev = handle.Write(yin);
|
|
auto dyin_dev = handle.Write(dy);
|
|
auto weights_dev = handle.Write(weights);
|
|
|
|
Workspace rspace{};
|
|
rspace.Write(reserveSpace);
|
|
|
|
std::vector<int> hlens(3, 0);
|
|
hlens[0] = nLayers * (dirMode != 0 ? 2 : 1);
|
|
hlens[1] = batch_seq[0];
|
|
hlens[2] = hiddenSize;
|
|
miopen::TensorDescriptor hiddenDesc(miopen::deref(rnnDesc).dataType, hlens);
|
|
|
|
std::vector<int> wlen(1, 0);
|
|
wlen[0] = weights.size();
|
|
miopen::TensorDescriptor weightDesc(miopen::deref(rnnDesc).dataType, wlen);
|
|
|
|
size_t in_sz = 0;
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, seqLength, inputDescs.data(), &in_sz);
|
|
std::vector<T> dx(in_sz / sizeof(T));
|
|
auto dx_dev = handle.Write(dx);
|
|
|
|
std::vector<T> dhx(initHidden.size());
|
|
auto dhx_dev = handle.Write(dhx);
|
|
|
|
auto dhy_dev = nodhy ? miopen::Allocator::ManageDataPtr{} : handle.Write(dhy);
|
|
auto hx_dev = nohx ? miopen::Allocator::ManageDataPtr{} : handle.Write(initHidden);
|
|
|
|
miopenRNNBackwardData(&handle,
|
|
rnnDesc,
|
|
seqLength,
|
|
outputDescs.data(),
|
|
yin_dev.get(),
|
|
outputDescs.data(),
|
|
dyin_dev.get(),
|
|
&hiddenDesc,
|
|
dhy_dev.get(),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
&weightDesc,
|
|
weights_dev.get(),
|
|
&hiddenDesc,
|
|
hx_dev.get(),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
inputDescs.data(),
|
|
dx_dev.get(),
|
|
&hiddenDesc,
|
|
((nodhx) ? nullptr : dhx_dev.get()),
|
|
&hiddenDesc,
|
|
nullptr,
|
|
wspace.ptr(),
|
|
wspace.size(),
|
|
rspace.ptr(),
|
|
rspace.size());
|
|
return std::make_tuple(handle.Read<T>(dx_dev, dx.size()),
|
|
(nodhx ? initHidden : handle.Read<T>(dhx_dev, dhx.size())),
|
|
rspace.Read<std::vector<T>>(),
|
|
wspace.Read<std::vector<T>>());
|
|
}
|
|
|
|
void fail() const
|
|
{
|
|
std::stringstream ss{};
|
|
ss << "./bin/MIOpenDriver rnn -n ";
|
|
for(int i = 0; i < seqLength; i++)
|
|
{
|
|
if(i < seqLength - 1)
|
|
{
|
|
ss << batch_seq.at(i) << ",";
|
|
}
|
|
else
|
|
{
|
|
ss << batch_seq.at(i);
|
|
}
|
|
}
|
|
ss << " -m gru -k " << seqLength << " -H " << hiddenSize << " -W " << inputVecLen << " -l "
|
|
<< nLayers << " -F 0 -r " << dirMode << " -b " << biasMode << " -p " << inputMode
|
|
<< std::endl;
|
|
ss << "inputMode: " << inputMode << " biasMode: " << biasMode << " dirMode: " << dirMode
|
|
<< std::endl;
|
|
ss << "hz: " << hiddenSize << " batch_n: " << batch_n << " seqLength: " << seqLength
|
|
<< " inputLen: " << inputVecLen << " numLayers: " << nLayers
|
|
<< " useDropout: " << int(use_dropout) << std::endl;
|
|
ss << "Backward Data GRU: " << std::endl;
|
|
GTEST_FAIL() << ss.str();
|
|
}
|
|
};
|
|
//~~~~~~~~~~~~ END BACKWARD DATA ~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
//****************************************************
|
|
// BACKWARDS WEIGHTS
|
|
//****************************************************
|
|
template <class T>
|
|
struct verify_backward_weights_gru
|
|
{
|
|
std::vector<T> input; // Y
|
|
std::vector<T> dy; // dY
|
|
std::vector<T> initHidden; // HX
|
|
std::vector<T> reserveSpace;
|
|
std::vector<T> workSpace;
|
|
std::vector<int> batch_seq;
|
|
int weightSize;
|
|
int hiddenSize;
|
|
int seqLength;
|
|
int nLayers;
|
|
bool biasMode{false};
|
|
bool dirMode{false};
|
|
int inputMode;
|
|
int batch_n;
|
|
int inputVecLen;
|
|
miopenRNNDescriptor_t rnnDesc;
|
|
size_t realHiddenSize;
|
|
bool nohx;
|
|
bool use_dropout;
|
|
|
|
verify_backward_weights_gru(miopenRNNDescriptor_t pRD,
|
|
const std::vector<T>& px,
|
|
const std::vector<T>& pdy,
|
|
const std::vector<T>& phx,
|
|
const std::vector<T>& pRS,
|
|
const std::vector<T>& pWS,
|
|
const std::vector<int>& pBS,
|
|
const int pHS,
|
|
const int pW,
|
|
const int pBN,
|
|
const int pS,
|
|
const int pNL,
|
|
const int pBM,
|
|
const int pDM,
|
|
const int pIM,
|
|
const int pVL,
|
|
const size_t pHXZ,
|
|
const bool pnohx = false,
|
|
const bool puse_dropout = false)
|
|
: input(px),
|
|
dy(pdy),
|
|
initHidden(phx),
|
|
reserveSpace(pRS),
|
|
workSpace(pWS),
|
|
batch_seq(pBS),
|
|
weightSize(pW),
|
|
hiddenSize(pHS),
|
|
seqLength(pS),
|
|
nLayers(pNL),
|
|
biasMode(pBM == 0 ? false : true),
|
|
dirMode(pDM == 0 ? false : true),
|
|
inputMode(pIM),
|
|
batch_n(pBN),
|
|
inputVecLen(pVL),
|
|
rnnDesc(pRD),
|
|
realHiddenSize(pHXZ),
|
|
nohx(pnohx),
|
|
use_dropout(puse_dropout)
|
|
{
|
|
if(!nohx)
|
|
initHidden = phx; // this may be intentionally a nullptr
|
|
else
|
|
initHidden.resize(realHiddenSize);
|
|
}
|
|
|
|
std::tuple<std::vector<T>> cpu()
|
|
{
|
|
int bi = dirMode ? 2 : 1;
|
|
std::vector<T> dweights(weightSize);
|
|
GRUBwdWeightCPUVerify(use_dropout,
|
|
input,
|
|
dweights, // (output) [ input_state_weight_trans
|
|
// hidden_state_weight0_trans
|
|
// input1_trans hidden1_trans ...
|
|
// output_weight; bidirectional
|
|
// reversed weights ]
|
|
initHidden, // initial hidden state
|
|
batch_seq, // input batch size
|
|
inputVecLen, // input data length
|
|
seqLength, // Number of iterations to unroll over
|
|
dirMode, // whether using bidirectional net
|
|
biasMode, // whether using bias
|
|
bi * nLayers, // 1 by numlayer (number of stacks of hidden
|
|
// layers) for unidirection, 2 by numlayer for
|
|
// bidirection
|
|
batch_seq.at(0), // equal to input batch size in_n[0]
|
|
hiddenSize, // hidden state number
|
|
inputMode,
|
|
reserveSpace,
|
|
workSpace,
|
|
nohx);
|
|
return std::make_tuple(dweights);
|
|
}
|
|
|
|
std::tuple<std::vector<T>> gpu()
|
|
{
|
|
auto&& handle = get_handle();
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, batch_seq, inputVecLen, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
batch_seq,
|
|
hiddenSize * ((dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
Workspace wspace{};
|
|
wspace.Write(workSpace);
|
|
Workspace rspace{};
|
|
rspace.Write(reserveSpace);
|
|
|
|
std::vector<T> dweights(weightSize);
|
|
auto dweights_dev = handle.Write(dweights);
|
|
miopen::TensorDescriptor weightDesc(miopen::deref(rnnDesc).dataType, {weightSize});
|
|
|
|
std::vector<int> hlens(3, 0);
|
|
hlens[0] = nLayers * (dirMode != 0 ? 2 : 1);
|
|
hlens[1] = batch_seq[0];
|
|
hlens[2] = hiddenSize;
|
|
miopen::TensorDescriptor hiddenDesc(miopen::deref(rnnDesc).dataType, hlens);
|
|
auto hx_dev = nohx ? miopen::Allocator::ManageDataPtr{} : handle.Write(initHidden);
|
|
auto dy_dev = handle.Write(dy);
|
|
auto input_dev = handle.Write(input);
|
|
|
|
miopenRNNBackwardWeights(&handle,
|
|
rnnDesc,
|
|
seqLength,
|
|
inputDescs.data(),
|
|
input_dev.get(),
|
|
&hiddenDesc,
|
|
hx_dev.get(),
|
|
outputDescs.data(),
|
|
dy_dev.get(),
|
|
&weightDesc,
|
|
dweights_dev.get(),
|
|
wspace.ptr(),
|
|
wspace.size(),
|
|
rspace.ptr(),
|
|
rspace.size());
|
|
auto retvec = handle.Read<T>(dweights_dev, dweights.size());
|
|
return std::make_tuple(retvec);
|
|
}
|
|
|
|
void fail() const
|
|
{
|
|
std::stringstream ss{};
|
|
ss << "./bin/MIOpenDriver rnn -n ";
|
|
for(int i = 0; i < seqLength; i++)
|
|
{
|
|
if(i < seqLength - 1)
|
|
{
|
|
ss << batch_seq.at(i) << ",";
|
|
}
|
|
else
|
|
{
|
|
ss << batch_seq.at(i);
|
|
}
|
|
}
|
|
ss << " -m gru -k " << seqLength << " -H " << hiddenSize << " -W " << inputVecLen << " -l "
|
|
<< nLayers << " -F 0 -r " << dirMode << " -b " << biasMode << " -p " << inputMode
|
|
<< std::endl;
|
|
ss << "inputMode: " << inputMode << " biasMode: " << biasMode << " dirMode: " << dirMode
|
|
<< std::endl;
|
|
ss << "hz: " << hiddenSize << " batch_n: " << batch_n << " seqLength: " << seqLength
|
|
<< " inputLen: " << inputVecLen << " numLayers: " << nLayers
|
|
<< " useDropout: " << int(use_dropout) << std::endl;
|
|
ss << "Backward Weights GRU: " << std::endl;
|
|
GTEST_FAIL() << ss.str();
|
|
}
|
|
};
|
|
//~~~~~~~~~~~~ END BACKWARD WEIGHTS ~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
struct TestCase
|
|
{
|
|
int batchSize{};
|
|
int seqLength{};
|
|
int inVecLen{};
|
|
int hiddenSize{};
|
|
int numLayers{};
|
|
int inputMode{};
|
|
int biasMode{};
|
|
int dirMode{};
|
|
bool useDropout = false;
|
|
bool nohx = false;
|
|
bool nodhy = false;
|
|
bool nohy = false;
|
|
bool nodhx = false;
|
|
bool flatBatchFill = false;
|
|
std::vector<int> batchSeq{};
|
|
};
|
|
|
|
using TestParam = std::tuple<int,
|
|
int,
|
|
int,
|
|
int,
|
|
int,
|
|
int,
|
|
int,
|
|
int,
|
|
bool,
|
|
bool,
|
|
bool,
|
|
bool,
|
|
bool,
|
|
bool,
|
|
std::vector<int>>;
|
|
|
|
TestCase ConvertParam(TestParam const& param)
|
|
{
|
|
auto [batchSize,
|
|
seqLength,
|
|
inVecLen,
|
|
hiddenSize,
|
|
numLayers,
|
|
inputMode,
|
|
biasMode,
|
|
dirMode,
|
|
useDropout,
|
|
nohx,
|
|
nodhy,
|
|
nohy,
|
|
nodhx,
|
|
flatBatchFill,
|
|
batchSeq] = param;
|
|
|
|
return TestCase{batchSize,
|
|
seqLength,
|
|
inVecLen,
|
|
hiddenSize,
|
|
numLayers,
|
|
inputMode,
|
|
biasMode,
|
|
dirMode,
|
|
useDropout,
|
|
nohx,
|
|
nodhy,
|
|
nohy,
|
|
nodhx,
|
|
flatBatchFill,
|
|
batchSeq};
|
|
}
|
|
|
|
std::vector<TestParam> deepbench_cases = {
|
|
{32, 1500, 216, 216, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 750, 286, 286, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 375, 286, 286, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 10, 2816, 2816, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 1500, 248, 248, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 12, 2048, 2048, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 1500, 156, 156, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 500, 156, 156, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 12, 1536, 1536, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 1500, 256, 256, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 500, 256, 256, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 10, 2560, 2560, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 1, 512, 512, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{32, 50, 1024, 1024, 1, 1, 0, 0, false, false, false, false, false, true, {0}},
|
|
{64, 50, 1024, 1024, 1, 1, 0, 0, false, false, false, false, false, true, {0}}};
|
|
|
|
std::vector<TestParam> extra_cases = {
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, true, false, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, false, true, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, true, true, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, true, false, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, false, true, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, true, true, false, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, false, false, false, true, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, false, false, true, true, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, false, false, true, false, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, false, false, false, true, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, false, false, true, true, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 0, false, true, true, true, true, false, {32, 32, 32}},
|
|
{32, 3, 128, 128, 1, 0, 0, 1, false, true, true, true, true, false, {32, 32, 32}}};
|
|
|
|
auto GenCases(bool gen_dropout)
|
|
{
|
|
std::vector<int> defaultBS(1);
|
|
|
|
std::vector<TestParam> cases{};
|
|
cases.push_back({17,
|
|
(gen_dropout ? 23 : 2),
|
|
13,
|
|
67,
|
|
(gen_dropout ? 3 : 1),
|
|
0,
|
|
0,
|
|
0,
|
|
gen_dropout,
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
defaultBS});
|
|
return ::testing::ValuesIn(cases);
|
|
}
|
|
|
|
auto GetFullBaseTests()
|
|
{
|
|
std::vector<int> modes(2, 0);
|
|
modes[1] = 1;
|
|
std::vector<int> defaultBS(1);
|
|
return ::testing::Combine(::testing::ValuesIn(get_gru_batchSize()),
|
|
::testing::ValuesIn(get_gru_seq_len()),
|
|
::testing::ValuesIn(get_gru_vector_len()),
|
|
::testing::ValuesIn(get_gru_hidden_size()),
|
|
::testing::ValuesIn(get_gru_num_layers()),
|
|
::testing::ValuesIn(modes),
|
|
::testing::ValuesIn(modes),
|
|
::testing::ValuesIn(modes),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({defaultBS}));
|
|
}
|
|
|
|
auto GetDropoutTests()
|
|
{
|
|
std::vector<int> modes(2, 0);
|
|
modes[1] = 1;
|
|
std::vector<int> defaultBS(1);
|
|
return ::testing::Combine(::testing::ValuesIn({17}),
|
|
::testing::ValuesIn({23}),
|
|
::testing::ValuesIn({13}),
|
|
::testing::ValuesIn({67}),
|
|
::testing::ValuesIn({3}),
|
|
::testing::ValuesIn({0}),
|
|
::testing::ValuesIn({0}),
|
|
::testing::ValuesIn(modes),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({false}),
|
|
::testing::ValuesIn({defaultBS}));
|
|
}
|
|
|
|
auto const& GetDropoutSmokeTests()
|
|
{
|
|
static auto cases = GenCases(true);
|
|
return cases;
|
|
}
|
|
|
|
auto const& GetDeepbenchTests() { return deepbench_cases; }
|
|
|
|
auto const& GetExtraTests() { return extra_cases; }
|
|
|
|
auto const& GetSmokeTests()
|
|
{
|
|
static auto cases = GenCases(false);
|
|
return cases;
|
|
}
|
|
|
|
template <class T>
|
|
class gru_test : public testing::TestWithParam<TestParam>
|
|
{
|
|
const double tolerance = 80; // Will be multiplied by std::numeric_limits<T>::epsilon()
|
|
TestCase param;
|
|
int batch_n{};
|
|
size_t statesSizeInBytes{0U};
|
|
|
|
public:
|
|
void fill_buffers(std::vector<T>& input, std::vector<T>& hx, std::vector<T>& weights)
|
|
{
|
|
auto fill_array_via_gen = [](std::vector<T>& dst, auto gen, int seed_offset = 0) {
|
|
prng::reset_seed(seed_offset);
|
|
size_t dst_sz = dst.size();
|
|
for(size_t it = 0; it < dst_sz; it++)
|
|
dst[it] = gen();
|
|
};
|
|
|
|
auto pos_gen = [](double scale, int range) {
|
|
return [=]() -> T { return prng::gen_descreet_unsigned<T>(scale, range); };
|
|
};
|
|
|
|
auto sign_gen = [](double scale, int range) {
|
|
return [=]() -> T { return prng::gen_descreet_uniform_sign<T>(scale, range); };
|
|
};
|
|
|
|
const double data_max_v = sqrt(1. / param.hiddenSize);
|
|
int data_range = 100;
|
|
const double data_scale = data_max_v / data_range;
|
|
fill_array_via_gen(input, pos_gen(data_scale, data_range), 0);
|
|
|
|
if(!param.nohx)
|
|
{
|
|
fill_array_via_gen(hx, pos_gen(data_scale, data_range), 1);
|
|
}
|
|
|
|
// filter
|
|
const double weights_max_v = sqrt(1. / param.hiddenSize);
|
|
int weights_range = 64;
|
|
const double weights_scale = weights_max_v / weights_range;
|
|
|
|
fill_array_via_gen(weights, sign_gen(weights_scale, weights_range), 2);
|
|
}
|
|
|
|
void fill_bwd_buffers(std::vector<T>& dy, std::vector<T>& dhy)
|
|
{
|
|
auto fill_array_via_gen = [](std::vector<T>& dst, auto gen, int seed_offset = 0) {
|
|
prng::reset_seed(seed_offset);
|
|
size_t dst_sz = dst.size();
|
|
|
|
for(size_t it = 0; it < dst_sz; it++)
|
|
dst[it] = gen();
|
|
};
|
|
|
|
auto sign_gen = [](double scale, int range) {
|
|
return [=]() { return prng::gen_descreet_uniform_sign<T>(scale, range); };
|
|
};
|
|
|
|
const double bwd_data_max_v = sqrt(1. / param.hiddenSize) / 8;
|
|
int bwd_data_range = 100;
|
|
const double bwd_data_scale = bwd_data_max_v / bwd_data_range;
|
|
|
|
if(!param.nodhy)
|
|
{
|
|
fill_array_via_gen(dhy, sign_gen(bwd_data_scale, bwd_data_range), 3);
|
|
}
|
|
|
|
fill_array_via_gen(dy, sign_gen(bwd_data_scale, bwd_data_range), 4);
|
|
prng::reset_seed();
|
|
}
|
|
|
|
void SetUp() override
|
|
{
|
|
prng::reset_seed();
|
|
param = ConvertParam(GetParam());
|
|
if(!param.useDropout)
|
|
{
|
|
param.batchSeq = generate_batchSeq(param.batchSize, param.seqLength)[0];
|
|
}
|
|
}
|
|
|
|
void Run()
|
|
{
|
|
if(param.batchSeq.empty() || 0 == param.batchSeq[0])
|
|
{
|
|
std::cout << "Empty batch sequence. Filling uniformly with batch size: "
|
|
<< param.batchSize << std::endl;
|
|
if(param.flatBatchFill)
|
|
{
|
|
param.batchSeq.clear();
|
|
param.batchSeq.resize(param.seqLength, param.batchSize);
|
|
}
|
|
else
|
|
{
|
|
param.batchSeq = GenBatchSeq(param.batchSize, param.seqLength);
|
|
}
|
|
}
|
|
|
|
if(param.batchSeq.size() != param.seqLength)
|
|
{
|
|
std::cerr << "FAILED: Batch sequence vector length, does not match sequence length."
|
|
<< std::endl;
|
|
GTEST_SKIP();
|
|
}
|
|
|
|
auto&& handle = get_handle();
|
|
|
|
batch_n = std::accumulate(param.batchSeq.begin(), param.batchSeq.end(), 0);
|
|
|
|
RNNDescGuard rnnDesc;
|
|
miopenRNNAlgo_t algoMode = miopenRNNdefault;
|
|
|
|
DropoutDescGuard DropoutDesc;
|
|
|
|
HandleGuard mio_handle;
|
|
void* dropout_state_buf = nullptr;
|
|
|
|
// See DestroyInternalRnnDropoutDesc — frees the descriptor allocated
|
|
// by miopenCreateRNNDescriptor that the upcoming Set* will leak.
|
|
DestroyInternalRnnDropoutDesc(rnnDesc);
|
|
|
|
if(param.useDropout)
|
|
{
|
|
mio_handle.create(handle.GetStream());
|
|
|
|
float dropout_rate = 0.5;
|
|
unsigned long long dropout_seed = 0ULL;
|
|
miopenDropoutGetStatesSize(mio_handle, &statesSizeInBytes);
|
|
|
|
(void)hipMalloc(static_cast<void**>(&dropout_state_buf), statesSizeInBytes);
|
|
|
|
miopenSetDropoutDescriptor(DropoutDesc,
|
|
mio_handle,
|
|
dropout_rate,
|
|
dropout_state_buf,
|
|
statesSizeInBytes,
|
|
dropout_seed,
|
|
false,
|
|
false,
|
|
MIOPEN_RNG_PSEUDO_XORWOW);
|
|
|
|
miopenSetRNNDescriptor_V2(rnnDesc,
|
|
param.hiddenSize,
|
|
param.numLayers,
|
|
DropoutDesc,
|
|
miopenRNNInputMode_t(param.inputMode),
|
|
miopenRNNDirectionMode_t(param.dirMode),
|
|
miopenGRU,
|
|
miopenRNNBiasMode_t(param.biasMode),
|
|
miopenRNNAlgo_t(algoMode),
|
|
miopen_type<T>{});
|
|
}
|
|
else
|
|
{
|
|
miopenSetRNNDescriptor(rnnDesc,
|
|
param.hiddenSize,
|
|
param.numLayers,
|
|
miopenRNNInputMode_t(param.inputMode),
|
|
miopenRNNDirectionMode_t(param.dirMode),
|
|
miopenGRU,
|
|
miopenRNNBiasMode_t(param.biasMode),
|
|
miopenRNNAlgo_t(algoMode),
|
|
miopen_type<T>{}); // defined in superclass testdriver
|
|
}
|
|
|
|
// Create input tensor
|
|
// If we are in skip mode, take the real input size to be the vector length.
|
|
auto inVecReal = (param.inputMode != 0) ? param.hiddenSize : param.inVecLen;
|
|
std::size_t in_sz = static_cast<std::size_t>(inVecReal) * batch_n;
|
|
std::size_t hx_sz = ((param.dirMode != 0) ? 2ULL : 1ULL) * param.hiddenSize *
|
|
param.batchSize * param.numLayers;
|
|
|
|
std::vector<T> input(in_sz), hx(hx_sz), dhyin(hx_sz);
|
|
|
|
size_t wei_bytes = [&]() {
|
|
size_t filter_bytes;
|
|
std::vector<int> inlens(2, 0);
|
|
inlens.at(0) = param.batchSeq.at(0);
|
|
inlens.at(1) = inVecReal;
|
|
auto firstInputDesc = miopen::TensorDescriptor(miopen::deref(rnnDesc).dataType, inlens);
|
|
miopenGetRNNParamsSize(
|
|
&handle, rnnDesc, &firstInputDesc, &filter_bytes, miopen::deref(rnnDesc).dataType);
|
|
return filter_bytes;
|
|
}();
|
|
|
|
auto wei_sz = wei_bytes / sizeof(T);
|
|
std::vector<T> weights(wei_sz);
|
|
|
|
std::vector<miopen::TensorDescriptor> inputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> inputDescs;
|
|
createTensorDescArray(
|
|
inputCPPDescs, inputDescs, param.batchSeq, inVecReal, miopen::deref(rnnDesc).dataType);
|
|
|
|
std::vector<miopen::TensorDescriptor> outputCPPDescs;
|
|
std::vector<miopenTensorDescriptor_t> outputDescs;
|
|
createTensorDescArray(outputCPPDescs,
|
|
outputDescs,
|
|
param.batchSeq,
|
|
param.hiddenSize * ((param.dirMode != 0) ? 2 : 1),
|
|
miopen::deref(rnnDesc).dataType);
|
|
|
|
size_t out_sz;
|
|
miopenGetRNNInputTensorSize(&handle, rnnDesc, param.seqLength, outputDescs.data(), &out_sz);
|
|
size_t reserveSpaceSize;
|
|
miopenGetRNNTrainingReserveSize(
|
|
&handle, rnnDesc, param.seqLength, inputDescs.data(), &reserveSpaceSize);
|
|
size_t workspace_size;
|
|
miopenGetRNNWorkspaceSize(
|
|
&handle, rnnDesc, param.seqLength, inputDescs.data(), &workspace_size);
|
|
|
|
size_t total_mem = statesSizeInBytes + reserveSpaceSize + workspace_size + 2 * out_sz +
|
|
(in_sz + wei_sz + (param.nohx ? 0 : hx_sz) + (param.nohy ? 0 : hx_sz) +
|
|
(param.nodhx ? 0 : hx_sz) + (param.nodhy ? 0 : hx_sz)) *
|
|
sizeof(T);
|
|
size_t device_mem = handle.GetGlobalMemorySize();
|
|
if(total_mem >= device_mem)
|
|
{
|
|
GTEST_SKIP() << "Config requires " << total_mem
|
|
<< " Bytes to write all necessary tensors to GPU. GPU has " << device_mem
|
|
<< " Bytes of memory." << std::endl;
|
|
}
|
|
|
|
fill_buffers(input, hx, weights);
|
|
|
|
auto fwdTrainOutputPair =
|
|
GruTestCompareResults(verify_forward_train_gru<T>{rnnDesc,
|
|
input,
|
|
hx,
|
|
weights,
|
|
param.batchSeq,
|
|
param.hiddenSize,
|
|
batch_n,
|
|
param.seqLength,
|
|
param.numLayers,
|
|
param.biasMode,
|
|
param.dirMode,
|
|
param.inputMode,
|
|
inVecReal,
|
|
hx_sz,
|
|
param.nohx,
|
|
param.nohy,
|
|
param.useDropout},
|
|
tolerance);
|
|
|
|
/// RETURNS std::make_tuple(output, hiddenState, reserveSpace);
|
|
auto yin = std::get<0>(fwdTrainOutputPair.second);
|
|
// auto curHiddenState = std::get<1>(fwdTrainOutputPair.second);
|
|
auto reserveSpaceFwdTrain = std::get<2>(fwdTrainOutputPair.second);
|
|
|
|
std::vector<T> dyin(yin.size());
|
|
|
|
fill_bwd_buffers(dyin, dhyin);
|
|
|
|
auto bwdDataOutputPair =
|
|
GruTestCompareResults(verify_backward_data_gru<T>{rnnDesc,
|
|
yin,
|
|
dyin,
|
|
dhyin,
|
|
hx,
|
|
weights,
|
|
reserveSpaceFwdTrain,
|
|
param.batchSeq,
|
|
param.hiddenSize,
|
|
batch_n,
|
|
param.seqLength,
|
|
param.numLayers,
|
|
param.biasMode,
|
|
param.dirMode,
|
|
param.inputMode,
|
|
inVecReal,
|
|
hx_sz,
|
|
param.nohx,
|
|
param.nodhy,
|
|
param.nodhx,
|
|
param.useDropout},
|
|
tolerance);
|
|
|
|
// RETURNS: std::make_tuple(dx, dhx, reserveSpace, workSpace);
|
|
auto reserveSpaceBwdData = std::get<2>(bwdDataOutputPair.second);
|
|
auto workSpaceBwdData = std::get<3>(bwdDataOutputPair.second);
|
|
// auto dweights_pair =
|
|
GruTestCompareResults(verify_backward_weights_gru<T>{rnnDesc,
|
|
input,
|
|
dyin,
|
|
hx,
|
|
reserveSpaceBwdData,
|
|
workSpaceBwdData,
|
|
param.batchSeq,
|
|
param.hiddenSize,
|
|
static_cast<int>(wei_sz),
|
|
batch_n,
|
|
param.seqLength,
|
|
param.numLayers,
|
|
param.biasMode,
|
|
param.dirMode,
|
|
param.inputMode,
|
|
inVecReal,
|
|
hx_sz,
|
|
param.nohx,
|
|
param.useDropout},
|
|
tolerance);
|
|
|
|
if(!param.useDropout)
|
|
{
|
|
GruTestCompareResults(verify_forward_infer_gru<T>{rnnDesc,
|
|
input,
|
|
hx,
|
|
weights,
|
|
param.batchSeq,
|
|
param.hiddenSize,
|
|
batch_n,
|
|
param.seqLength,
|
|
param.numLayers,
|
|
param.biasMode,
|
|
param.dirMode,
|
|
param.inputMode,
|
|
inVecReal,
|
|
hx_sz,
|
|
param.nohx,
|
|
param.nohy},
|
|
tolerance);
|
|
}
|
|
// DLOWELL: Subtracting delta weights may produce NAN and infinities. Further investigation
|
|
// is needed.
|
|
// auto dweights = std::get<1>(dweights_pair);
|
|
// std::transform(weightData.begin( ), weightData.end( ), dweights.begin( ),
|
|
// weightData.begin( ),std::minus<T>( ));
|
|
// GruTestCompareResults(verify_forward_infer_gru<T>{rnnDesc, inputData,
|
|
// curHiddenState, curCellState, weightData,
|
|
// batchSeq,
|
|
// hiddenSize, batch_n,
|
|
// seqLength, numLayers,
|
|
// biasMode, dirMode,
|
|
// inputMode, inVecReal});
|
|
|
|
// Free the DropoutDescriptor that miopenSetRNNDescriptor just allocated.
|
|
// In the dropout path, the internal pointer aliases the user-owned
|
|
// DropoutDescGuard — freeing it would double-free.
|
|
if(!param.useDropout)
|
|
DestroyInternalRnnDropoutDesc(rnnDesc);
|
|
|
|
if(param.useDropout)
|
|
(void)hipFree(dropout_state_buf);
|
|
}
|
|
};
|
|
|
|
struct TestNameGenerator
|
|
{
|
|
std::string operator()(const ::testing::TestParamInfo<TestParam>& param_info)
|
|
{
|
|
std::stringstream ss{};
|
|
auto print_bool = [](bool value) { return value ? "true" : "false"; };
|
|
|
|
auto print_batch_seq = [](std::vector<int> const& vec) {
|
|
std::stringstream vec_ss{};
|
|
for(auto el : vec)
|
|
{
|
|
vec_ss << std::to_string(el) << "_";
|
|
}
|
|
return vec_ss.str();
|
|
};
|
|
|
|
auto param = ConvertParam(param_info.param);
|
|
|
|
ss << "batchSize_" << param.batchSize << "_seqLength_" << param.seqLength << "_inVecLen_"
|
|
<< param.inVecLen << "_hiddenSize_" << param.hiddenSize << "_numLayers_"
|
|
<< param.numLayers << "_inputMode_" << param.inputMode << "_biasMode_" << param.biasMode
|
|
<< "_dirMode_" << param.dirMode << "_useDropout_" << print_bool(param.useDropout)
|
|
<< "_nohx_" << print_bool(param.nohx) << "_nodhy_" << print_bool(param.nodhy) << "_nohy_"
|
|
<< print_bool(param.nohy) << "_nodhx_" << print_bool(param.nodhx) << "_flatBatchFill_"
|
|
<< print_bool(param.flatBatchFill) << "_batch_seq_" << print_batch_seq(param.batchSeq);
|
|
return ss.str();
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
struct gru_dropout_test : public gru_test<T>
|
|
{
|
|
// intentionally empty
|
|
};
|
|
|
|
template <typename T>
|
|
struct gru_deepbench_test : public gru_test<T>
|
|
{
|
|
// intentionally empty
|
|
};
|
|
|
|
template <typename T>
|
|
struct gru_extra_test : public gru_test<T>
|
|
{
|
|
// intentionally empty
|
|
};
|
|
|
|
} // anonymous namespace
|
|
|
|
using GPU_GRU_Base_FP32 = gru_test<float>;
|
|
using GPU_GRU_Base_FP16 = gru_test<half_float::half>;
|
|
|
|
using GPU_GRU_Dropout_FP32 = gru_dropout_test<float>;
|
|
using GPU_GRU_Dropout_FP16 = gru_dropout_test<half_float::half>;
|
|
|
|
using GPU_GRU_Deepbench_FP32 = gru_deepbench_test<float>;
|
|
using GPU_GRU_Deepbench_FP16 = gru_deepbench_test<half_float::half>;
|
|
|
|
using GPU_GRU_Extra_FP32 = gru_extra_test<float>;
|
|
using GPU_GRU_Extra_FP16 = gru_extra_test<half_float::half>;
|
|
|
|
TEST_P(GPU_GRU_Base_FP32, TestFloat32) { Run(); }
|
|
TEST_P(GPU_GRU_Base_FP16, TestFloat16) { Run(); }
|
|
|
|
// Base tests
|
|
INSTANTIATE_TEST_SUITE_P(Full, GPU_GRU_Base_FP32, GetFullBaseTests(), TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Smoke, GPU_GRU_Base_FP32, GetSmokeTests(), TestNameGenerator{});
|
|
|
|
INSTANTIATE_TEST_SUITE_P(Full, GPU_GRU_Base_FP16, GetFullBaseTests(), TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Smoke, GPU_GRU_Base_FP16, GetSmokeTests(), TestNameGenerator{});
|
|
|
|
// Dropout tests
|
|
TEST_P(GPU_GRU_Dropout_FP32, TestFloat32) { Run(); }
|
|
TEST_P(GPU_GRU_Dropout_FP16, TestFloat16) { Run(); }
|
|
|
|
INSTANTIATE_TEST_SUITE_P(Full, GPU_GRU_Dropout_FP32, GetDropoutTests(), TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Smoke, GPU_GRU_Dropout_FP32, GetDropoutSmokeTests(), TestNameGenerator{});
|
|
|
|
INSTANTIATE_TEST_SUITE_P(Full, GPU_GRU_Dropout_FP16, GetDropoutTests(), TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Smoke, GPU_GRU_Dropout_FP16, GetDropoutSmokeTests(), TestNameGenerator{});
|
|
|
|
// // Deepbench tests
|
|
TEST_P(GPU_GRU_Deepbench_FP32, TestFloat32) { Run(); }
|
|
TEST_P(GPU_GRU_Deepbench_FP16, TestFloat16) { Run(); }
|
|
|
|
INSTANTIATE_TEST_SUITE_P(Full,
|
|
GPU_GRU_Deepbench_FP32,
|
|
::testing::ValuesIn(GetDeepbenchTests()),
|
|
TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Full,
|
|
GPU_GRU_Deepbench_FP16,
|
|
::testing::ValuesIn(GetDeepbenchTests()),
|
|
TestNameGenerator{});
|
|
|
|
// Extra tests
|
|
TEST_P(GPU_GRU_Extra_FP32, TestFloat32) { Run(); }
|
|
TEST_P(GPU_GRU_Extra_FP16, TestFloat16) { Run(); }
|
|
|
|
INSTANTIATE_TEST_SUITE_P(Full,
|
|
GPU_GRU_Extra_FP32,
|
|
::testing::ValuesIn(GetExtraTests()),
|
|
TestNameGenerator{});
|
|
INSTANTIATE_TEST_SUITE_P(Full,
|
|
GPU_GRU_Extra_FP16,
|
|
::testing::ValuesIn(GetExtraTests()),
|
|
TestNameGenerator{});
|