derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
4445 lines
170 KiB
C++
4445 lines
170 KiB
C++
// Copyright (C) 2023 - 2026 Advanced Micro Devices, Inc. All rights reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
// THE SOFTWARE.
|
|
|
|
#ifndef FFT_PARAMS_H
|
|
#define FFT_PARAMS_H
|
|
|
|
#include <algorithm>
|
|
#include <hip/hip_runtime.h>
|
|
#include <iostream>
|
|
#include <mutex>
|
|
#include <numeric>
|
|
#include <sstream>
|
|
#ifdef _OPENMP
|
|
#include <omp.h>
|
|
#endif
|
|
#include <random>
|
|
#include <tuple>
|
|
#include <unordered_set>
|
|
#include <valarray>
|
|
#include <vector>
|
|
|
|
#ifdef ROCFFT_MPI_ENABLE
|
|
#include <mpi.h>
|
|
#endif
|
|
|
|
#include "../shared/arithmetic.h"
|
|
#include "../shared/array_validator.h"
|
|
#include "../shared/client_data_layout_helpers.h"
|
|
#include "../shared/client_except.h"
|
|
#include "../shared/data_gen_device.h"
|
|
#include "../shared/data_gen_host.h"
|
|
#include "../shared/device_properties.h"
|
|
#include "../shared/fft_enums.h"
|
|
#include "../shared/gpubuf.h"
|
|
#include "../shared/printbuffer.h"
|
|
#include "../shared/ptrdiff.h"
|
|
#include "../shared/rocfft_complex.h"
|
|
|
|
// Used for CLI11 parsing of precision enum
|
|
static bool lexical_cast(const std::string& word, fft_precision& precision)
|
|
{
|
|
if(word == "half")
|
|
precision = fft_precision_half;
|
|
else if(word == "single")
|
|
precision = fft_precision_single;
|
|
else if(word == "double")
|
|
precision = fft_precision_double;
|
|
else
|
|
throw std::runtime_error("Invalid precision specified");
|
|
return true;
|
|
}
|
|
|
|
// Used for CLI11 parsing of auto-allocation enum
|
|
static bool lexical_cast(const std::string& word, fft_auto_allocation& auto_allocation)
|
|
{
|
|
if(word == "on")
|
|
auto_allocation = fft_auto_allocation_on;
|
|
else if(word == "off")
|
|
auto_allocation = fft_auto_allocation_off;
|
|
else if(word == "default")
|
|
auto_allocation = fft_auto_allocation_default;
|
|
else
|
|
throw std::runtime_error(
|
|
"Invalid auto-allocation behavior specified (choose \"on\", \"off\", or \"default\")");
|
|
return true;
|
|
}
|
|
|
|
// Used for CLI11 parsing of input gen enum
|
|
static bool lexical_cast(const std::string& word, fft_input_generator& gen)
|
|
{
|
|
if(word == "0")
|
|
gen = fft_input_random_generator_device;
|
|
else if(word == "1")
|
|
gen = fft_input_random_generator_host;
|
|
else if(word == "2")
|
|
gen = fft_input_generator_device;
|
|
else if(word == "3")
|
|
gen = fft_input_generator_host;
|
|
else
|
|
throw std::runtime_error("Invalid input generator specified");
|
|
#ifndef USE_HIPRAND
|
|
if(gen == fft_input_random_generator_device)
|
|
throw std::runtime_error(
|
|
"Device random input generation is not available, as hipRAND support is not enabled");
|
|
#endif
|
|
return true;
|
|
}
|
|
|
|
// Determine the size of the data type given the precision and type.
|
|
template <typename Tsize>
|
|
inline Tsize var_size(const fft_precision precision, const fft_array_type type)
|
|
{
|
|
size_t var_size = 0;
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
var_size = sizeof(rocfft_fp16);
|
|
break;
|
|
case fft_precision_single:
|
|
var_size = sizeof(float);
|
|
break;
|
|
case fft_precision_double:
|
|
var_size = sizeof(double);
|
|
break;
|
|
}
|
|
if(array_type_is_interleaved(type))
|
|
var_size *= 2;
|
|
return var_size;
|
|
}
|
|
|
|
// Given an array type and transform length, strides, etc, initialize
|
|
// values into the input device buffer.
|
|
//
|
|
// length/istride/batch/dist describe the physical layout of the
|
|
// input buffer. The buffer is treated as a sub-brick of a field,
|
|
// though the brick may cover the entire field.
|
|
//
|
|
// Lower coordinate of the brick in the field is provided by
|
|
// field_lower (FFT dimension coordinate) and field_lower_batch
|
|
// (batch dimension coordinate). For a brick that covers the whole
|
|
// field, these are all zeroes.
|
|
//
|
|
// field_contig_stride + dist are the field's stride and dist if the
|
|
// field were contiguous.
|
|
template <typename Tfloat, typename Tint1>
|
|
inline void set_input(std::vector<gpubuf>& input,
|
|
const fft_input_generator igen,
|
|
const fft_array_type itype,
|
|
const std::vector<size_t>& length,
|
|
const std::vector<size_t>& ilength,
|
|
const std::vector<size_t>& istride,
|
|
const std::vector<size_t>& ioffset,
|
|
const Tint1& whole_length,
|
|
const Tint1& whole_stride,
|
|
const size_t idist,
|
|
const size_t nbatch,
|
|
const hipDeviceProp_t& deviceProp,
|
|
const Tint1& field_lower,
|
|
const size_t field_lower_batch,
|
|
const Tint1& field_contig_stride,
|
|
const size_t field_contig_dist)
|
|
{
|
|
#ifndef USE_HIPRAND
|
|
if(igen == fft_input_random_generator_device)
|
|
throw std::runtime_error(
|
|
"Device random input generation is not available, as hipRAND support is not enabled");
|
|
#endif // USE_HIPRAND
|
|
|
|
if(is_host_generator(igen))
|
|
throw std::runtime_error("Host input generation is not available for gpu buffers");
|
|
|
|
auto isize = count_iters(whole_length) * nbatch;
|
|
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
auto ibuffer = (rocfft_complex<Tfloat>*)input[0].data() + ioffset[0];
|
|
#ifdef USE_HIPRAND
|
|
if(igen == fft_input_generator_device)
|
|
generate_interleaved_data(
|
|
whole_length, idist, isize, whole_stride, nbatch, ibuffer, deviceProp);
|
|
else if(igen == fft_input_random_generator_device)
|
|
generate_random_interleaved_data(whole_length,
|
|
idist,
|
|
isize,
|
|
whole_stride,
|
|
ibuffer,
|
|
deviceProp,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
#else
|
|
generate_interleaved_data(
|
|
whole_length, idist, isize, whole_stride, nbatch, ibuffer, deviceProp);
|
|
#endif // USE_HIPRAND
|
|
|
|
if(itype == fft_array_type_hermitian_interleaved)
|
|
{
|
|
auto ibuffer_2 = (rocfft_complex<Tfloat>*)input[0].data();
|
|
impose_hermitian_symmetry_interleaved(
|
|
length, ilength, istride, idist, nbatch, ibuffer_2, deviceProp);
|
|
}
|
|
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
{
|
|
auto ibuffer_real = (Tfloat*)input[0].data() + ioffset[0];
|
|
auto ibuffer_imag = (Tfloat*)input[1].data() + ioffset[1];
|
|
|
|
#ifdef USE_HIPRAND
|
|
if(igen == fft_input_generator_device)
|
|
generate_planar_data(whole_length,
|
|
idist,
|
|
isize,
|
|
whole_stride,
|
|
nbatch,
|
|
ibuffer_real,
|
|
ibuffer_imag,
|
|
deviceProp);
|
|
else if(igen == fft_input_random_generator_device)
|
|
generate_random_planar_data(whole_length,
|
|
idist,
|
|
isize,
|
|
whole_stride,
|
|
ibuffer_real,
|
|
ibuffer_imag,
|
|
deviceProp,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
#else
|
|
generate_planar_data(whole_length,
|
|
idist,
|
|
isize,
|
|
whole_stride,
|
|
nbatch,
|
|
ibuffer_real,
|
|
ibuffer_imag,
|
|
deviceProp);
|
|
#endif // USE_HIPRAND
|
|
|
|
if(itype == fft_array_type_hermitian_planar)
|
|
impose_hermitian_symmetry_planar(
|
|
length, ilength, istride, idist, nbatch, ibuffer_real, ibuffer_imag, deviceProp);
|
|
|
|
break;
|
|
}
|
|
case fft_array_type_real:
|
|
{
|
|
auto ibuffer = (Tfloat*)input[0].data() + ioffset[0];
|
|
|
|
#ifdef USE_HIPRAND
|
|
if(igen == fft_input_generator_device)
|
|
generate_real_data(
|
|
whole_length, idist, isize, whole_stride, nbatch, ibuffer, deviceProp);
|
|
else if(igen == fft_input_random_generator_device)
|
|
generate_random_real_data(whole_length,
|
|
idist,
|
|
isize,
|
|
whole_stride,
|
|
ibuffer,
|
|
deviceProp,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
#else
|
|
generate_real_data(whole_length, idist, isize, whole_stride, nbatch, ibuffer, deviceProp);
|
|
#endif // USE_HIPRAND
|
|
|
|
break;
|
|
}
|
|
default:
|
|
throw std::runtime_error("Input layout format not yet supported");
|
|
}
|
|
}
|
|
|
|
// Given an array type and transform length, strides, etc, initialize
|
|
// values into the input host buffer.
|
|
//
|
|
// length/istride/batch/dist describe the physical layout of the
|
|
// input buffer. The buffer is treated as a sub-brick of a field,
|
|
// though the brick may cover the entire field.
|
|
//
|
|
// Lower coordinate of the brick in the field is provided by
|
|
// field_lower (FFT dimension coordinate) and field_lower_batch
|
|
// (batch dimension coordinate). For a brick that covers the whole
|
|
// field, these are all zeroes.
|
|
//
|
|
// field_contig_stride + dist are the field's stride and dist if the
|
|
// field were contiguous.
|
|
template <typename Tfloat, typename Tint1>
|
|
inline void set_input(std::vector<hostbuf>& input,
|
|
const fft_input_generator igen,
|
|
const fft_array_type itype,
|
|
const std::vector<size_t>& length,
|
|
const std::vector<size_t>& ilength,
|
|
const std::vector<size_t>& istride,
|
|
const std::vector<size_t>& ioffset,
|
|
const Tint1& whole_length,
|
|
const Tint1& whole_stride,
|
|
const size_t idist,
|
|
const size_t nbatch,
|
|
const hipDeviceProp_t& deviceProp,
|
|
const Tint1 field_lower,
|
|
const size_t field_lower_batch,
|
|
const Tint1 field_contig_stride,
|
|
const size_t field_contig_dist)
|
|
{
|
|
if(is_device_generator(igen))
|
|
throw std::runtime_error(
|
|
"Device random input generation is not available for host buffers");
|
|
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
if(igen == fft_input_generator_host)
|
|
generate_interleaved_data<Tfloat>(
|
|
input, ioffset, whole_length, whole_stride, idist, nbatch);
|
|
else if(igen == fft_input_random_generator_host)
|
|
generate_random_interleaved_data<Tfloat>(input,
|
|
ioffset,
|
|
whole_length,
|
|
whole_stride,
|
|
idist,
|
|
nbatch,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
|
|
if(itype == fft_array_type_hermitian_interleaved)
|
|
impose_hermitian_symmetry_interleaved<Tfloat>(
|
|
input, ioffset, length, istride, idist, nbatch);
|
|
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
{
|
|
if(igen == fft_input_generator_host)
|
|
generate_planar_data<Tfloat>(input, ioffset, whole_length, whole_stride, idist, nbatch);
|
|
else if(igen == fft_input_random_generator_host)
|
|
generate_random_planar_data<Tfloat>(input,
|
|
ioffset,
|
|
whole_length,
|
|
whole_stride,
|
|
idist,
|
|
nbatch,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
|
|
if(itype == fft_array_type_hermitian_planar)
|
|
impose_hermitian_symmetry_planar<Tfloat>(
|
|
input, ioffset, length, istride, idist, nbatch);
|
|
|
|
break;
|
|
}
|
|
case fft_array_type_real:
|
|
{
|
|
if(igen == fft_input_generator_host)
|
|
generate_real_data<Tfloat>(input, ioffset, whole_length, whole_stride, idist, nbatch);
|
|
else if(igen == fft_input_random_generator_host)
|
|
generate_random_real_data<Tfloat>(input,
|
|
ioffset,
|
|
whole_length,
|
|
whole_stride,
|
|
idist,
|
|
nbatch,
|
|
field_lower,
|
|
field_lower_batch,
|
|
field_contig_stride,
|
|
field_contig_dist);
|
|
|
|
break;
|
|
}
|
|
default:
|
|
throw std::runtime_error("Input layout format not yet supported");
|
|
}
|
|
}
|
|
|
|
// unroll set_input for dimension 1, 2, 3
|
|
template <typename Tbuff, typename Tfloat>
|
|
inline void set_input(std::vector<Tbuff>& input,
|
|
const fft_input_generator igen,
|
|
const fft_array_type itype,
|
|
const std::vector<size_t>& length,
|
|
const std::vector<size_t>& ilength,
|
|
const std::vector<size_t>& istride,
|
|
const std::vector<size_t>& ioffset,
|
|
const size_t idist,
|
|
const size_t nbatch,
|
|
const hipDeviceProp_t& deviceProp,
|
|
const std::vector<size_t>& field_lower,
|
|
const size_t field_lower_batch,
|
|
const std::vector<size_t>& field_contig_stride,
|
|
const size_t field_contig_dist)
|
|
{
|
|
switch(length.size())
|
|
{
|
|
case 1:
|
|
set_input<Tfloat, size_t>(input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength,
|
|
istride,
|
|
ioffset,
|
|
ilength[0],
|
|
istride[0],
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
field_lower[0],
|
|
field_lower_batch,
|
|
field_contig_stride[0],
|
|
field_contig_dist);
|
|
break;
|
|
case 2:
|
|
set_input<Tfloat, std::tuple<size_t, size_t>>(
|
|
input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength,
|
|
istride,
|
|
ioffset,
|
|
std::make_tuple(ilength[0], ilength[1]),
|
|
std::make_tuple(istride[0], istride[1]),
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
std::make_tuple(field_lower[0], field_lower[1]),
|
|
field_lower_batch,
|
|
std::make_tuple(field_contig_stride[0], field_contig_stride[1]),
|
|
field_contig_dist);
|
|
break;
|
|
case 3:
|
|
set_input<Tfloat, std::tuple<size_t, size_t, size_t>>(
|
|
input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength,
|
|
istride,
|
|
ioffset,
|
|
std::make_tuple(ilength[0], ilength[1], ilength[2]),
|
|
std::make_tuple(istride[0], istride[1], istride[2]),
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
std::make_tuple(field_lower[0], field_lower[1], field_lower[2]),
|
|
field_lower_batch,
|
|
std::make_tuple(field_contig_stride[0], field_contig_stride[1], field_contig_stride[2]),
|
|
field_contig_dist);
|
|
break;
|
|
default:
|
|
abort();
|
|
}
|
|
}
|
|
|
|
// Container class for test parameters.
|
|
class fft_params
|
|
{
|
|
public:
|
|
// All parameters are row-major.
|
|
std::vector<size_t> length;
|
|
std::vector<size_t> istride;
|
|
std::vector<size_t> ostride;
|
|
size_t nbatch = 1;
|
|
fft_precision precision = fft_precision_single;
|
|
fft_transform_type transform_type = fft_transform_type_complex_forward;
|
|
fft_result_placement placement = fft_placement_inplace;
|
|
size_t idist = 0;
|
|
size_t odist = 0;
|
|
fft_array_type itype = fft_array_type_unset;
|
|
fft_array_type otype = fft_array_type_unset;
|
|
std::vector<size_t> ioffset = {0, 0};
|
|
std::vector<size_t> ooffset = {0, 0};
|
|
|
|
std::vector<size_t> isize;
|
|
std::vector<size_t> osize;
|
|
|
|
#ifdef USE_HIPRAND
|
|
fft_input_generator igen = fft_input_random_generator_device;
|
|
#else
|
|
fft_input_generator igen = fft_input_random_generator_host;
|
|
#endif
|
|
|
|
fft_auto_allocation auto_allocate = fft_auto_allocation_default;
|
|
|
|
// JIT callback parameters are specified at plan creation time, so
|
|
// they need to be known and remembered before create_plan() is
|
|
// called
|
|
struct jit_cb_state_t
|
|
{
|
|
const char* symbol = nullptr;
|
|
std::vector<char> func;
|
|
std::vector<gpubuf> data;
|
|
size_t shared_mem_bytes = 0;
|
|
// "convert" data to std::vector<void*> as needed in APIs
|
|
inline std::vector<void*> get_raw_data_ptrs() const
|
|
{
|
|
std::vector<void*> ret;
|
|
ret.reserve(data.size());
|
|
for(auto& buf : data)
|
|
ret.push_back(buf.data());
|
|
return ret;
|
|
}
|
|
|
|
// throw if this state is not usable (symbol/code/data missing,
|
|
// etc)
|
|
void check_valid() const
|
|
{
|
|
if(!symbol)
|
|
throw std::invalid_argument("missing JIT symbol");
|
|
if(func.empty())
|
|
throw std::invalid_argument("missing JIT code");
|
|
// data can be empty if the callback function doesn't need
|
|
// it, but if nonempty must have one ptr per device
|
|
if(!data.empty()
|
|
&& data.size() != static_cast<size_t>(rocfft_scoped_device::device_count()))
|
|
throw std::invalid_argument("invalid number of JIT data ptrs");
|
|
}
|
|
};
|
|
std::shared_ptr<jit_cb_state_t> load_jit_cb_state;
|
|
std::shared_ptr<jit_cb_state_t> store_jit_cb_state;
|
|
|
|
// Check that JIT callback parameters have been specified properly,
|
|
// if JIT callbacks are required. Throws an exception if the check
|
|
// fails.
|
|
void check_jit_callback_state() const
|
|
{
|
|
if(run_callbacks != fft_callback_type_jit)
|
|
return;
|
|
|
|
if(!load_jit_cb_state)
|
|
throw std::invalid_argument("missing JIT load state");
|
|
load_jit_cb_state->check_valid();
|
|
if(!store_jit_cb_state)
|
|
throw std::invalid_argument("missing JIT store state");
|
|
store_jit_cb_state->check_valid();
|
|
}
|
|
|
|
enum fft_mp_lib
|
|
{
|
|
fft_mp_lib_none,
|
|
fft_mp_lib_mpi,
|
|
};
|
|
fft_mp_lib mp_lib = fft_mp_lib_none;
|
|
// Pointer to a library-specific communicator type. Note that this
|
|
// is a pointer, so whatever this points to must live as long as
|
|
// this pointer does.
|
|
void* mp_comm = nullptr;
|
|
|
|
struct fft_brick
|
|
{
|
|
// all vectors here are row-major, with same length as FFT
|
|
// dimension + 1 (for batch dimension)
|
|
|
|
// inclusive lower bound of brick
|
|
std::vector<size_t> lower;
|
|
// exclusive upper bound of brick
|
|
std::vector<size_t> upper;
|
|
// stride of brick in memory
|
|
std::vector<size_t> stride;
|
|
|
|
// compute the length of this brick
|
|
std::vector<size_t> length() const
|
|
{
|
|
std::vector<size_t> ret;
|
|
for(size_t i = 0; i < lower.size(); ++i)
|
|
ret.push_back(upper[i] - lower[i]);
|
|
return ret;
|
|
}
|
|
|
|
// compute offset of lower bound in a field with the given
|
|
// stride + dist (batch stride is separate)
|
|
size_t lower_field_offset(std::vector<size_t> stride, size_t dist) const
|
|
{
|
|
// brick strides include batch, so adjust our input accordingly
|
|
stride.insert(stride.begin(), dist);
|
|
|
|
return std::inner_product(lower.begin(), lower.end(), stride.begin(), 0);
|
|
}
|
|
|
|
// location of the brick
|
|
int rank = 0;
|
|
int device = 0;
|
|
|
|
bool operator==(const fft_brick& other) const
|
|
{
|
|
return this->lower == other.lower && this->upper == other.upper
|
|
&& this->stride == other.stride && this->rank == other.rank
|
|
&& this->device == other.device;
|
|
}
|
|
bool operator!=(const fft_brick& other) const
|
|
{
|
|
return !(*this == other);
|
|
}
|
|
};
|
|
|
|
struct fft_field
|
|
{
|
|
std::vector<fft_brick> bricks;
|
|
|
|
// Brick ordering is significant - a rank needs to pass
|
|
// pointers to execute() in the order that the bricks were
|
|
// added to the field.
|
|
//
|
|
// But we also want to sort the bricks by rank, so that we can
|
|
// efficiently find the bricks for a given rank. So do a
|
|
// stable sort to preserve the relative order of bricks on any
|
|
// rank, even after they're sorted.
|
|
void stable_sort_by_rank()
|
|
{
|
|
std::stable_sort(
|
|
bricks.begin(), bricks.end(), [](const fft_brick& a, const fft_brick& b) {
|
|
return a.rank < b.rank;
|
|
});
|
|
}
|
|
};
|
|
|
|
// heuristic algorithm to create a 3D grid that covers a field starting in index (0,0,0),
|
|
// this approach intends to minimize the surface area of field bricks
|
|
void set_default_3d_grid(unsigned int mp_ranks,
|
|
std::vector<size_t> const grid_dims,
|
|
std::vector<unsigned int>& fft_grid)
|
|
{
|
|
std::valarray<unsigned int> global_indices = {static_cast<unsigned int>(grid_dims[0]),
|
|
static_cast<unsigned int>(grid_dims[1]),
|
|
static_cast<unsigned int>(grid_dims[2])};
|
|
|
|
// set initial grid as ones
|
|
std::valarray<unsigned int> selected_grid = {1, 1, 1};
|
|
|
|
// helper method to compute the surface of a brick
|
|
auto surface = [&](std::valarray<unsigned int> const& proc_grid) -> unsigned int {
|
|
auto brick_size = global_indices / proc_grid;
|
|
return (brick_size * brick_size.cshift(1)).sum();
|
|
};
|
|
|
|
unsigned int selected_surface = std::numeric_limits<unsigned int>::max();
|
|
|
|
for(unsigned int i = 1; i <= mp_ranks; i++)
|
|
{
|
|
if(mp_ranks % i == 0)
|
|
{
|
|
unsigned int remainder = mp_ranks / i;
|
|
for(unsigned int j = 1; j <= remainder; j++)
|
|
{
|
|
if(remainder % j == 0)
|
|
{
|
|
std::valarray<unsigned int> grid = {i, j, remainder / j};
|
|
unsigned int const surf = surface(grid);
|
|
if(surf < selected_surface)
|
|
{
|
|
selected_surface = surf;
|
|
selected_grid = grid;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(selected_grid[0] * selected_grid[1] * selected_grid[2] != mp_ranks)
|
|
{
|
|
throw std::runtime_error("Grid dimensions do not multiply to mp_ranks.");
|
|
}
|
|
|
|
fft_grid = {selected_grid[0], selected_grid[1], selected_grid[2]};
|
|
}
|
|
|
|
// heuristic algorithm to create a 2D grid that covers a field starting in index (0,0),
|
|
// this approach intends to minimize the surface area of field bricks
|
|
void set_default_2d_grid(unsigned int mp_ranks,
|
|
std::vector<size_t> const grid_dims,
|
|
std::vector<unsigned int>& fft_grid)
|
|
{
|
|
std::valarray<unsigned int> global_indices
|
|
= {static_cast<unsigned int>(grid_dims[0]), static_cast<unsigned int>(grid_dims[1])};
|
|
|
|
// set initial grid as ones
|
|
std::valarray<unsigned int> selected_grid = {1, 1};
|
|
|
|
// helper method to compute the surface of a brick
|
|
auto surface = [&](std::valarray<unsigned int> const& proc_grid) -> unsigned int {
|
|
auto brick_size = global_indices / proc_grid;
|
|
return (brick_size * brick_size.cshift(1)).sum();
|
|
};
|
|
|
|
unsigned int selected_surface = std::numeric_limits<unsigned int>::max();
|
|
|
|
for(unsigned int i = 1; i <= mp_ranks; i++)
|
|
{
|
|
if(mp_ranks % i == 0)
|
|
{
|
|
std::valarray<unsigned int> grid = {i, mp_ranks / i};
|
|
unsigned int const surf = surface(grid);
|
|
if(surf < selected_surface)
|
|
{
|
|
selected_surface = surf;
|
|
selected_grid = grid;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(selected_grid[0] * selected_grid[1] != mp_ranks)
|
|
{
|
|
throw std::runtime_error("Grid dimensions do not multiply to mp_ranks.");
|
|
}
|
|
|
|
fft_grid = {selected_grid[0], selected_grid[1]};
|
|
}
|
|
|
|
void set_default_grid(const int& mp_ranks,
|
|
std::vector<unsigned int>& ingrid,
|
|
std::vector<unsigned int>& outgrid)
|
|
{
|
|
if(ingrid.empty())
|
|
{
|
|
if(length.size() == 3)
|
|
{
|
|
set_default_3d_grid(mp_ranks, length, ingrid);
|
|
}
|
|
else if(length.size() == 2)
|
|
{
|
|
set_default_2d_grid(mp_ranks, length, ingrid);
|
|
}
|
|
else if(length.size() == 1)
|
|
{
|
|
ingrid.push_back(mp_ranks);
|
|
}
|
|
}
|
|
if(outgrid.empty())
|
|
{
|
|
if(length.size() == 3)
|
|
{
|
|
set_default_3d_grid(mp_ranks, length, outgrid);
|
|
}
|
|
else if(length.size() == 2)
|
|
{
|
|
set_default_2d_grid(mp_ranks, length, outgrid);
|
|
}
|
|
else if(length.size() == 1)
|
|
{
|
|
outgrid.push_back(mp_ranks);
|
|
}
|
|
}
|
|
|
|
// sanity checks
|
|
int ingrid_size = product(ingrid.begin(), ingrid.end());
|
|
int outgrid_size = product(outgrid.begin(), outgrid.end());
|
|
|
|
if((ingrid.size() != length.size()) || (outgrid.size() != length.size()))
|
|
{
|
|
throw std::runtime_error(
|
|
"Grid of processors must be of the same dimension as the FFT!");
|
|
}
|
|
|
|
if((ingrid_size != mp_ranks) || (outgrid_size != mp_ranks))
|
|
{
|
|
throw std::runtime_error("Number of GPUs defined by input/output grids must be "
|
|
"equal to the number of available GPUs!");
|
|
}
|
|
}
|
|
|
|
// optional brick decomposition of inputs/outputs
|
|
std::vector<fft_field> ifields;
|
|
std::vector<fft_field> ofields;
|
|
|
|
// Check that a supplied vector of callback pointers has an
|
|
// expected size. Optionally also check that each pointer is
|
|
// non-null. Throws an exception if a check fails. The vector
|
|
// itself can be null, as callbacks are optional.
|
|
static void check_callback_vec(const std::vector<void*>* cb, size_t expected_size, bool nonnull)
|
|
{
|
|
if(!cb)
|
|
return;
|
|
if(cb->size() != expected_size)
|
|
throw std::invalid_argument("expected " + std::to_string(expected_size)
|
|
+ " callback pointers, got " + std::to_string(cb->size()));
|
|
if(nonnull && std::any_of(cb->begin(), cb->end(), [](void* p) { return p == nullptr; }))
|
|
throw std::invalid_argument("null callback function");
|
|
}
|
|
|
|
// simple "multi-GPU" count, meaning the library decides on the
|
|
// decomposition instead of it being explicit as bricks. only
|
|
// has an effect if set to a number > 1.
|
|
size_t multiGPU = 0;
|
|
|
|
// run testing load/store callbacks
|
|
fft_callback_type run_callbacks = fft_callback_type_none;
|
|
static constexpr double load_cb_scalar = 0.457813941;
|
|
static constexpr double store_cb_scalar = 0.391504938;
|
|
|
|
// Check that data outside of output strides is not overwritten.
|
|
// This is only set explicitly on some tests where there's space
|
|
// between dimensions, but the dimensions are still in-order.
|
|
// We're not trying to generically find holes in arbitrary data
|
|
// layouts.
|
|
//
|
|
// NOTE: this flag is not included in tokens, since it doesn't
|
|
// affect how the FFT library behaves.
|
|
bool check_output_strides = false;
|
|
|
|
// scaling factor - we do a pointwise multiplication of outputs by
|
|
// this factor
|
|
double scale_factor = 1.0;
|
|
|
|
fft_params(){};
|
|
virtual ~fft_params(){};
|
|
|
|
// copying and moving
|
|
fft_params(const fft_params&) = default;
|
|
fft_params& operator=(const fft_params&) = default;
|
|
fft_params(fft_params&&) = default;
|
|
fft_params& operator=(fft_params&&) = default;
|
|
|
|
virtual void setup() {}
|
|
virtual void cleanup() {}
|
|
|
|
// Given an array type, return the name as a string.
|
|
static std::string array_type_name(const fft_array_type type, bool verbose = true)
|
|
{
|
|
switch(type)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
return verbose ? "fft_array_type_complex_interleaved" : "CI";
|
|
case fft_array_type_complex_planar:
|
|
return verbose ? "fft_array_type_complex_planar" : "CP";
|
|
case fft_array_type_real:
|
|
return verbose ? "fft_array_type_real" : "R";
|
|
case fft_array_type_hermitian_interleaved:
|
|
return verbose ? "fft_array_type_hermitian_interleaved" : "HI";
|
|
case fft_array_type_hermitian_planar:
|
|
return verbose ? "fft_array_type_hermitian_planar" : "HP";
|
|
case fft_array_type_unset:
|
|
return verbose ? "fft_array_type_unset" : "UN";
|
|
}
|
|
return "";
|
|
}
|
|
|
|
std::string transform_type_name() const
|
|
{
|
|
switch(transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
return "fft_transform_type_complex_forward";
|
|
case fft_transform_type_complex_inverse:
|
|
return "fft_transform_type_complex_inverse";
|
|
case fft_transform_type_real_forward:
|
|
return "fft_transform_type_real_forward";
|
|
case fft_transform_type_real_inverse:
|
|
return "fft_transform_type_real_inverse";
|
|
default:
|
|
throw std::runtime_error("Invalid transform type");
|
|
}
|
|
}
|
|
|
|
bool is_inverse() const
|
|
{
|
|
return is_bwd(transform_type);
|
|
}
|
|
|
|
bool is_forward() const
|
|
{
|
|
return is_fwd(transform_type);
|
|
}
|
|
|
|
// Convert to string for output.
|
|
std::string str(const std::string& separator = ", ") const
|
|
{
|
|
// top-level stride/dist are not used when fields are specified.
|
|
const bool have_ifields = !ifields.empty();
|
|
const bool have_ofields = !ofields.empty();
|
|
|
|
std::stringstream ss;
|
|
auto print_size_vec = [&](const char* description, const std::vector<size_t>& vec) {
|
|
ss << description << ":";
|
|
for(auto i : vec)
|
|
ss << " " << i;
|
|
ss << separator;
|
|
};
|
|
auto print_fields = [&](const char* description, const std::vector<fft_field>& fields) {
|
|
for(unsigned int fidx = 0; fidx < fields.size(); ++fidx)
|
|
{
|
|
const auto& f = fields[fidx];
|
|
ss << description << " " << fidx << ":" << separator;
|
|
for(unsigned int bidx = 0; bidx < f.bricks.size(); ++bidx)
|
|
{
|
|
const auto& b = f.bricks[bidx];
|
|
ss << " brick " << bidx << ":" << separator;
|
|
print_size_vec(" lower", b.lower);
|
|
print_size_vec(" upper", b.upper);
|
|
print_size_vec(" stride", b.stride);
|
|
ss << " device: " << b.device << separator;
|
|
}
|
|
}
|
|
};
|
|
|
|
print_size_vec("length", length);
|
|
if(have_ifields)
|
|
{
|
|
print_fields("ifield", ifields);
|
|
}
|
|
else
|
|
{
|
|
print_size_vec("istride", istride);
|
|
ss << "idist: " << idist << separator;
|
|
}
|
|
|
|
if(have_ofields)
|
|
{
|
|
print_fields("ofield", ofields);
|
|
}
|
|
else
|
|
{
|
|
print_size_vec("ostride", ostride);
|
|
ss << "odist: " << odist << separator;
|
|
}
|
|
|
|
ss << "batch: " << nbatch << separator;
|
|
print_size_vec("isize", isize);
|
|
print_size_vec("osize", osize);
|
|
|
|
print_size_vec("ioffset", ioffset);
|
|
print_size_vec("ooffset", ooffset);
|
|
|
|
if(placement == fft_placement_inplace)
|
|
ss << "in-place";
|
|
else
|
|
ss << "out-of-place";
|
|
ss << separator;
|
|
ss << "transform_type: " << transform_type_name() << separator;
|
|
ss << array_type_name(itype) << " -> " << array_type_name(otype) << separator;
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
ss << "half-precision";
|
|
break;
|
|
case fft_precision_single:
|
|
ss << "single-precision";
|
|
break;
|
|
case fft_precision_double:
|
|
ss << "double-precision";
|
|
break;
|
|
}
|
|
ss << separator;
|
|
|
|
print_size_vec("ilength", ilength());
|
|
print_size_vec("olength", olength());
|
|
|
|
print_size_vec("ibuffer_size", ibuffer_sizes());
|
|
print_size_vec("obuffer_size", obuffer_sizes());
|
|
|
|
if(scale_factor != 1.0)
|
|
ss << "scale factor: " << scale_factor << separator;
|
|
|
|
return ss.str();
|
|
}
|
|
|
|
// Produce a stringified token of the test fft params.
|
|
std::string token() const
|
|
{
|
|
std::string ret;
|
|
|
|
switch(transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
ret += "complex_forward_";
|
|
break;
|
|
case fft_transform_type_complex_inverse:
|
|
ret += "complex_inverse_";
|
|
break;
|
|
case fft_transform_type_real_forward:
|
|
ret += "real_forward_";
|
|
break;
|
|
case fft_transform_type_real_inverse:
|
|
ret += "real_inverse_";
|
|
break;
|
|
}
|
|
|
|
auto append_size_vec = [&ret](const std::vector<size_t>& vec) {
|
|
for(auto s : vec)
|
|
{
|
|
ret += "_";
|
|
ret += std::to_string(s);
|
|
}
|
|
};
|
|
|
|
ret += "len";
|
|
append_size_vec(length);
|
|
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
ret += "_half_";
|
|
break;
|
|
case fft_precision_single:
|
|
ret += "_single_";
|
|
break;
|
|
case fft_precision_double:
|
|
ret += "_double_";
|
|
break;
|
|
}
|
|
|
|
switch(placement)
|
|
{
|
|
case fft_placement_inplace:
|
|
ret += "ip_";
|
|
break;
|
|
case fft_placement_notinplace:
|
|
ret += "op_";
|
|
break;
|
|
}
|
|
|
|
ret += "batch_";
|
|
ret += std::to_string(nbatch);
|
|
|
|
auto append_array_type = [&ret](fft_array_type type) {
|
|
switch(type)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
ret += "CI";
|
|
break;
|
|
case fft_array_type_complex_planar:
|
|
ret += "CP";
|
|
break;
|
|
case fft_array_type_real:
|
|
ret += "R";
|
|
break;
|
|
case fft_array_type_hermitian_interleaved:
|
|
ret += "HI";
|
|
break;
|
|
case fft_array_type_hermitian_planar:
|
|
ret += "HP";
|
|
break;
|
|
default:
|
|
ret += "UN";
|
|
break;
|
|
}
|
|
};
|
|
|
|
auto append_brick_info = [&ret, &append_size_vec](const fft_brick& b) {
|
|
ret += "_brick";
|
|
|
|
ret += "_lower";
|
|
append_size_vec(b.lower);
|
|
ret += "_upper";
|
|
append_size_vec(b.upper);
|
|
ret += "_stride";
|
|
append_size_vec(b.stride);
|
|
if(b.rank)
|
|
{
|
|
ret += "_rank_";
|
|
ret += std::to_string(b.rank);
|
|
}
|
|
ret += "_dev_";
|
|
ret += std::to_string(b.device);
|
|
};
|
|
|
|
const bool have_ifields = !ifields.empty();
|
|
const bool have_ofields = !ofields.empty();
|
|
|
|
if(have_ifields)
|
|
{
|
|
for(const auto& f : ifields)
|
|
{
|
|
ret += "_ifield";
|
|
for(const auto& b : f.bricks)
|
|
append_brick_info(b);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ret += "_istride";
|
|
append_size_vec(istride);
|
|
ret += "_";
|
|
append_array_type(itype);
|
|
}
|
|
|
|
if(have_ofields)
|
|
{
|
|
for(const auto& f : ofields)
|
|
{
|
|
ret += "_ofield";
|
|
for(const auto& b : f.bricks)
|
|
append_brick_info(b);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ret += "_ostride";
|
|
append_size_vec(ostride);
|
|
ret += "_";
|
|
append_array_type(otype);
|
|
}
|
|
|
|
if(!have_ifields)
|
|
{
|
|
ret += "_idist_";
|
|
ret += std::to_string(idist);
|
|
}
|
|
if(!have_ofields)
|
|
{
|
|
ret += "_odist_";
|
|
ret += std::to_string(odist);
|
|
}
|
|
|
|
if(!have_ifields)
|
|
{
|
|
ret += "_ioffset";
|
|
append_size_vec(ioffset);
|
|
}
|
|
|
|
if(!have_ofields)
|
|
{
|
|
ret += "_ooffset";
|
|
append_size_vec(ooffset);
|
|
}
|
|
|
|
switch(run_callbacks)
|
|
{
|
|
case fft_callback_type_funcptr:
|
|
ret += "_CB";
|
|
break;
|
|
case fft_callback_type_jit:
|
|
ret += "_JITCB";
|
|
break;
|
|
case fft_callback_type_none:
|
|
break;
|
|
}
|
|
|
|
if(scale_factor != 1.0)
|
|
ret += "_scale";
|
|
|
|
if(multiGPU > 1)
|
|
{
|
|
ret += "_multigpu_";
|
|
ret += std::to_string(multiGPU);
|
|
}
|
|
|
|
if(auto_allocate != fft_auto_allocation_default)
|
|
{
|
|
ret += "_autoallocation_";
|
|
ret += (auto_allocate == fft_auto_allocation_on ? "on" : "off");
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
// Set all params from a stringified token.
|
|
void from_token(std::string token)
|
|
{
|
|
std::vector<std::string> vals;
|
|
|
|
std::string delimiter = "_";
|
|
{
|
|
size_t pos = 0;
|
|
while((pos = token.find(delimiter)) != std::string::npos)
|
|
{
|
|
auto val = token.substr(0, pos);
|
|
vals.push_back(val);
|
|
token.erase(0, pos + delimiter.length());
|
|
}
|
|
vals.push_back(token);
|
|
}
|
|
|
|
auto size_parser
|
|
= [](const std::vector<std::string>& vals, const std::string token, size_t& pos) {
|
|
if(vals[pos++] != token)
|
|
throw std::runtime_error("Unable to parse token");
|
|
return std::stoull(vals[pos++]);
|
|
};
|
|
|
|
auto vector_parser
|
|
= [](const std::vector<std::string>& vals, const std::string token, size_t& pos) {
|
|
if(vals[pos++] != token)
|
|
throw std::runtime_error("Unable to parse token");
|
|
std::vector<size_t> vec;
|
|
|
|
while(pos < vals.size())
|
|
{
|
|
if(std::all_of(vals[pos].begin(), vals[pos].end(), ::isdigit))
|
|
{
|
|
vec.push_back(std::stoull(vals[pos++]));
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
return vec;
|
|
};
|
|
|
|
auto type_parser = [](const std::string& val) {
|
|
if(val == "CI")
|
|
return fft_array_type_complex_interleaved;
|
|
else if(val == "CP")
|
|
return fft_array_type_complex_planar;
|
|
else if(val == "R")
|
|
return fft_array_type_real;
|
|
else if(val == "HI")
|
|
return fft_array_type_hermitian_interleaved;
|
|
else if(val == "HP")
|
|
return fft_array_type_hermitian_planar;
|
|
return fft_array_type_unset;
|
|
};
|
|
|
|
auto field_parser = [&vector_parser, &size_parser](const std::vector<std::string>& vals,
|
|
size_t& pos,
|
|
std::vector<fft_field>& output) {
|
|
// skip over ifield/ofield word
|
|
pos++;
|
|
fft_field& f = output.emplace_back();
|
|
while(pos < vals.size() && vals[pos] == "brick")
|
|
{
|
|
fft_brick& b = f.bricks.emplace_back();
|
|
pos++;
|
|
b.lower = vector_parser(vals, "lower", pos);
|
|
b.upper = vector_parser(vals, "upper", pos);
|
|
b.stride = vector_parser(vals, "stride", pos);
|
|
if(vals[pos] == "rank")
|
|
b.rank = size_parser(vals, "rank", pos);
|
|
b.device = size_parser(vals, "dev", pos);
|
|
}
|
|
};
|
|
|
|
size_t pos = 0;
|
|
|
|
bool complex = vals[pos++] == "complex";
|
|
bool forward = vals[pos++] == "forward";
|
|
|
|
if(complex && forward)
|
|
transform_type = fft_transform_type_complex_forward;
|
|
if(complex && !forward)
|
|
transform_type = fft_transform_type_complex_inverse;
|
|
if(!complex && forward)
|
|
transform_type = fft_transform_type_real_forward;
|
|
if(!complex && !forward)
|
|
transform_type = fft_transform_type_real_inverse;
|
|
|
|
length = vector_parser(vals, "len", pos);
|
|
|
|
if(vals[pos] == "half")
|
|
precision = fft_precision_half;
|
|
else if(vals[pos] == "single")
|
|
precision = fft_precision_single;
|
|
else if(vals[pos] == "double")
|
|
precision = fft_precision_double;
|
|
pos++;
|
|
|
|
placement = (vals[pos++] == "ip") ? fft_placement_inplace : fft_placement_notinplace;
|
|
|
|
nbatch = size_parser(vals, "batch", pos);
|
|
|
|
// strides, bricks etc are mixed in from here, so just keep
|
|
// looking at the next token to decide what to do
|
|
while(pos < vals.size() - 1)
|
|
{
|
|
const auto& next_token = vals[pos];
|
|
if(next_token == "istride")
|
|
{
|
|
istride = vector_parser(vals, "istride", pos);
|
|
itype = type_parser(vals[pos]);
|
|
pos++;
|
|
}
|
|
else if(next_token == "ostride")
|
|
{
|
|
ostride = vector_parser(vals, "ostride", pos);
|
|
otype = type_parser(vals[pos]);
|
|
pos++;
|
|
}
|
|
else if(next_token == "idist")
|
|
idist = size_parser(vals, "idist", pos);
|
|
else if(next_token == "odist")
|
|
odist = size_parser(vals, "odist", pos);
|
|
else if(next_token == "ioffset")
|
|
ioffset = vector_parser(vals, "ioffset", pos);
|
|
else if(next_token == "ooffset")
|
|
ooffset = vector_parser(vals, "ooffset", pos);
|
|
else if(next_token == "ifield")
|
|
field_parser(vals, pos, ifields);
|
|
else if(next_token == "ofield")
|
|
field_parser(vals, pos, ofields);
|
|
else
|
|
break;
|
|
}
|
|
|
|
if(pos < vals.size() && vals[pos] == "CB")
|
|
{
|
|
run_callbacks = fft_callback_type_funcptr;
|
|
++pos;
|
|
}
|
|
|
|
if(pos < vals.size() && vals[pos] == "JITCB")
|
|
{
|
|
run_callbacks = fft_callback_type_jit;
|
|
++pos;
|
|
}
|
|
|
|
if(pos < vals.size() && vals[pos] == "scale")
|
|
{
|
|
// just pick some factor that's not zero or one
|
|
scale_factor = 0.1239;
|
|
++pos;
|
|
}
|
|
|
|
if(pos < vals.size() && vals[pos] == "multigpu")
|
|
{
|
|
++pos;
|
|
multiGPU = std::stoull(vals[pos++]);
|
|
}
|
|
|
|
auto_allocate = fft_auto_allocation_default; // default if unspecified
|
|
if(pos < vals.size() && vals[pos] == "autoallocation")
|
|
{
|
|
++pos;
|
|
lexical_cast(vals[pos++], auto_allocate);
|
|
}
|
|
}
|
|
|
|
// Stream output operator (for gtest, etc).
|
|
friend std::ostream& operator<<(std::ostream& stream, const fft_params& params)
|
|
{
|
|
stream << params.str();
|
|
return stream;
|
|
}
|
|
|
|
// Dimension of the transform.
|
|
size_t dim() const
|
|
{
|
|
return length.size();
|
|
}
|
|
|
|
virtual std::vector<size_t> ilength() const
|
|
{
|
|
auto ilength = length;
|
|
if(transform_type == fft_transform_type_real_inverse)
|
|
ilength[dim() - 1] = ilength[dim() - 1] / 2 + 1;
|
|
return ilength;
|
|
}
|
|
|
|
virtual std::vector<size_t> olength() const
|
|
{
|
|
auto olength = length;
|
|
if(transform_type == fft_transform_type_real_forward)
|
|
olength[dim() - 1] = olength[dim() - 1] / 2 + 1;
|
|
return olength;
|
|
}
|
|
|
|
static size_t nbuffer(const fft_array_type type)
|
|
{
|
|
switch(type)
|
|
{
|
|
case fft_array_type_real:
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
return 1;
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
return 2;
|
|
case fft_array_type_unset:
|
|
return 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Number of input buffers
|
|
size_t nibuffer() const
|
|
{
|
|
return nbuffer(itype);
|
|
}
|
|
|
|
// Number of output buffers
|
|
size_t nobuffer() const
|
|
{
|
|
return nbuffer(otype);
|
|
}
|
|
|
|
void set_iotypes()
|
|
{
|
|
if(itype == fft_array_type_unset)
|
|
{
|
|
switch(transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
case fft_transform_type_complex_inverse:
|
|
itype = fft_array_type_complex_interleaved;
|
|
break;
|
|
case fft_transform_type_real_forward:
|
|
itype = fft_array_type_real;
|
|
break;
|
|
case fft_transform_type_real_inverse:
|
|
itype = fft_array_type_hermitian_interleaved;
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid transform type");
|
|
}
|
|
}
|
|
if(otype == fft_array_type_unset)
|
|
{
|
|
switch(transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
case fft_transform_type_complex_inverse:
|
|
otype = fft_array_type_complex_interleaved;
|
|
break;
|
|
case fft_transform_type_real_forward:
|
|
otype = fft_array_type_hermitian_interleaved;
|
|
break;
|
|
case fft_transform_type_real_inverse:
|
|
otype = fft_array_type_real;
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid transform type");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check that the input and output types are consistent.
|
|
bool check_iotypes() const
|
|
{
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_interleaved:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid Input array type format");
|
|
}
|
|
|
|
switch(otype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_interleaved:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid Input array type format");
|
|
}
|
|
|
|
// Check that format choices are supported
|
|
if(transform_type != fft_transform_type_real_forward
|
|
&& transform_type != fft_transform_type_real_inverse)
|
|
{
|
|
if(placement == fft_placement_inplace && itype != otype)
|
|
{
|
|
throw std::runtime_error(
|
|
"In-place transforms must have identical input and output types");
|
|
}
|
|
}
|
|
|
|
bool okformat = true;
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_complex_planar:
|
|
okformat = (otype == fft_array_type_complex_interleaved
|
|
|| otype == fft_array_type_complex_planar);
|
|
break;
|
|
case fft_array_type_hermitian_interleaved:
|
|
case fft_array_type_hermitian_planar:
|
|
okformat = otype == fft_array_type_real;
|
|
break;
|
|
case fft_array_type_real:
|
|
okformat = (otype == fft_array_type_hermitian_interleaved
|
|
|| otype == fft_array_type_hermitian_planar);
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid Input array type format");
|
|
}
|
|
|
|
return okformat;
|
|
}
|
|
|
|
// Given a length vector, set the rest of the strides.
|
|
// The optional argument stride0 sets the stride for the contiguous dimension.
|
|
// The optional rcpadding argument sets the stride correctly for in-place
|
|
// multi-dimensional real/complex transforms.
|
|
// Format is row-major.
|
|
template <typename T1>
|
|
std::vector<T1> compute_stride(const std::vector<T1>& length,
|
|
const std::vector<size_t>& stride0 = std::vector<size_t>(),
|
|
const bool rcpadding = false) const
|
|
{
|
|
std::vector<T1> stride(dim());
|
|
|
|
size_t dimoffset = 0;
|
|
|
|
if(stride0.size() == 0)
|
|
{
|
|
// Set the contiguous stride:
|
|
stride[dim() - 1] = 1;
|
|
dimoffset = 1;
|
|
}
|
|
else
|
|
{
|
|
// Copy the input values to the end of the stride array:
|
|
for(size_t i = 0; i < stride0.size(); ++i)
|
|
{
|
|
stride[dim() - stride0.size() + i] = stride0[i];
|
|
}
|
|
}
|
|
|
|
if(stride0.size() < dim())
|
|
{
|
|
// Compute any remaining values via recursion.
|
|
for(size_t i = dim() - dimoffset - stride0.size(); i-- > 0;)
|
|
{
|
|
auto lengthip1 = length[i + 1];
|
|
if(rcpadding && i == dim() - 2)
|
|
{
|
|
lengthip1 = 2 * (lengthip1 / 2 + 1);
|
|
}
|
|
stride[i] = stride[i + 1] * lengthip1;
|
|
}
|
|
}
|
|
|
|
return stride;
|
|
}
|
|
|
|
void compute_istride()
|
|
{
|
|
istride = compute_stride(ilength(),
|
|
istride,
|
|
placement == fft_placement_inplace
|
|
&& transform_type == fft_transform_type_real_forward);
|
|
}
|
|
|
|
void compute_ostride()
|
|
{
|
|
ostride = compute_stride(olength(),
|
|
ostride,
|
|
placement == fft_placement_inplace
|
|
&& transform_type == fft_transform_type_real_inverse);
|
|
}
|
|
|
|
virtual void compute_isize()
|
|
{
|
|
auto il = ilength();
|
|
size_t val = compute_ptrdiff(il, istride, nbatch, idist);
|
|
isize.resize(nibuffer());
|
|
for(unsigned int i = 0; i < isize.size(); ++i)
|
|
{
|
|
isize[i] = val + ioffset[i];
|
|
}
|
|
}
|
|
|
|
virtual void compute_osize()
|
|
{
|
|
auto ol = olength();
|
|
size_t val = compute_ptrdiff(ol, ostride, nbatch, odist);
|
|
osize.resize(nobuffer());
|
|
for(unsigned int i = 0; i < osize.size(); ++i)
|
|
{
|
|
osize[i] = val + ooffset[i];
|
|
}
|
|
}
|
|
|
|
std::vector<size_t> ibuffer_sizes() const
|
|
{
|
|
std::vector<size_t> ibuffer_sizes;
|
|
|
|
// In-place real-to-complex transforms need to have enough space in the input buffer to
|
|
// accommodate the output, which is slightly larger.
|
|
if(placement == fft_placement_inplace && transform_type == fft_transform_type_real_forward)
|
|
{
|
|
return obuffer_sizes();
|
|
}
|
|
|
|
if(isize.empty())
|
|
return ibuffer_sizes;
|
|
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
ibuffer_sizes.resize(2);
|
|
break;
|
|
default:
|
|
ibuffer_sizes.resize(1);
|
|
}
|
|
for(unsigned i = 0; i < ibuffer_sizes.size(); i++)
|
|
{
|
|
ibuffer_sizes[i] = isize[i] * var_size<size_t>(precision, itype);
|
|
}
|
|
return ibuffer_sizes;
|
|
}
|
|
|
|
virtual std::vector<size_t> obuffer_sizes() const
|
|
{
|
|
std::vector<size_t> obuffer_sizes;
|
|
|
|
if(osize.empty())
|
|
return obuffer_sizes;
|
|
|
|
switch(otype)
|
|
{
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
obuffer_sizes.resize(2);
|
|
break;
|
|
default:
|
|
obuffer_sizes.resize(1);
|
|
}
|
|
for(unsigned i = 0; i < obuffer_sizes.size(); i++)
|
|
{
|
|
obuffer_sizes[i] = osize[i] * var_size<size_t>(precision, otype);
|
|
}
|
|
return obuffer_sizes;
|
|
}
|
|
|
|
// Compute the idist for a given transform based on the placeness, transform type, and data
|
|
// layout.
|
|
size_t compute_idist() const
|
|
{
|
|
size_t dist = 0;
|
|
// In-place 1D transforms need extra dist.
|
|
if(transform_type == fft_transform_type_real_forward && dim() == 1
|
|
&& placement == fft_placement_inplace)
|
|
{
|
|
dist = 2 * (length[0] / 2 + 1) * istride[0];
|
|
return dist;
|
|
}
|
|
|
|
if(transform_type == fft_transform_type_real_inverse && dim() == 1)
|
|
{
|
|
dist = (length[0] / 2 + 1) * istride[0];
|
|
return dist;
|
|
}
|
|
|
|
dist = (transform_type == fft_transform_type_real_inverse)
|
|
? (length[dim() - 1] / 2 + 1) * istride[dim() - 1]
|
|
: length[dim() - 1] * istride[dim() - 1];
|
|
for(unsigned int i = 0; i < dim() - 1; ++i)
|
|
{
|
|
dist = std::max(length[i] * istride[i], dist);
|
|
}
|
|
return dist;
|
|
}
|
|
void set_idist()
|
|
{
|
|
if(idist != 0)
|
|
return;
|
|
idist = compute_idist();
|
|
}
|
|
|
|
// Compute the odist for a given transform based on the placeness, transform type, and data
|
|
// layout. Row-major.
|
|
size_t compute_odist() const
|
|
{
|
|
size_t dist = 0;
|
|
// In-place 1D transforms need extra dist.
|
|
if(transform_type == fft_transform_type_real_inverse && dim() == 1
|
|
&& placement == fft_placement_inplace)
|
|
{
|
|
dist = 2 * (length[0] / 2 + 1) * ostride[0];
|
|
return dist;
|
|
}
|
|
|
|
if(transform_type == fft_transform_type_real_forward && dim() == 1)
|
|
{
|
|
dist = (length[0] / 2 + 1) * ostride[0];
|
|
return dist;
|
|
}
|
|
|
|
dist = (transform_type == fft_transform_type_real_forward)
|
|
? (length[dim() - 1] / 2 + 1) * ostride[dim() - 1]
|
|
: length[dim() - 1] * ostride[dim() - 1];
|
|
for(unsigned int i = 0; i < dim() - 1; ++i)
|
|
{
|
|
dist = std::max(length[i] * ostride[i], dist);
|
|
}
|
|
return dist;
|
|
}
|
|
void set_odist()
|
|
{
|
|
if(odist != 0)
|
|
return;
|
|
odist = compute_odist();
|
|
}
|
|
|
|
// Put the length, stride, batch, and dist into a single length/stride array and pass off to the
|
|
// validity checker.
|
|
bool valid_length_stride_batch_dist(const std::vector<size_t>& l0,
|
|
const std::vector<size_t>& s0,
|
|
const size_t n,
|
|
const size_t dist,
|
|
const int verbose = 0) const
|
|
{
|
|
if(l0.size() != s0.size())
|
|
return false;
|
|
|
|
// Length and stride vectors, including bathes:
|
|
std::vector<size_t> l{}, s{};
|
|
for(unsigned int i = 0; i < l0.size(); ++i)
|
|
{
|
|
if(l0[i] > 1)
|
|
{
|
|
if(s0[i] == 0)
|
|
return false;
|
|
l.push_back(l0[i]);
|
|
s.push_back(s0[i]);
|
|
}
|
|
}
|
|
if(n > 1)
|
|
{
|
|
if(dist == 0)
|
|
return false;
|
|
l.push_back(n);
|
|
s.push_back(dist);
|
|
}
|
|
|
|
return array_valid(l, s, verbose);
|
|
}
|
|
|
|
// Return true if the given GPU parameters would produce a valid transform.
|
|
bool valid(const int verbose = 0) const
|
|
{
|
|
if(ioffset.size() < nibuffer() || ooffset.size() < nobuffer())
|
|
return false;
|
|
|
|
// Check that in-place transforms have the same input and output stride:
|
|
if(placement == fft_placement_inplace)
|
|
{
|
|
const auto stridesize = std::min(istride.size(), ostride.size());
|
|
bool samestride = true;
|
|
for(unsigned int i = 0; i < stridesize; ++i)
|
|
{
|
|
// (strides are irrelevant for unit lengths)
|
|
if(istride[i] != ostride[i] && length[i] > 1)
|
|
samestride = false;
|
|
}
|
|
if((transform_type == fft_transform_type_complex_forward
|
|
|| transform_type == fft_transform_type_complex_inverse)
|
|
&& !samestride)
|
|
{
|
|
// In-place transforms require identical input and output strides.
|
|
if(verbose)
|
|
{
|
|
std::cout << "istride:";
|
|
for(const auto& i : istride)
|
|
std::cout << " " << i;
|
|
std::cout << " ostride:";
|
|
for(const auto& i : ostride)
|
|
std::cout << " " << i;
|
|
std::cout << " differ; skipped for in-place transforms: skipping test"
|
|
<< std::endl;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
if((transform_type == fft_transform_type_complex_forward
|
|
|| transform_type == fft_transform_type_complex_inverse)
|
|
&& (idist != odist) && nbatch > 1)
|
|
{
|
|
// In-place transforms require identical distance, if
|
|
// batch > 1. If batch is 1 then dist is ignored and
|
|
// the FFT should still work.
|
|
if(verbose)
|
|
{
|
|
std::cout << "idist:" << idist << " odist:" << odist
|
|
<< " differ; skipped for in-place transforms: skipping test"
|
|
<< std::endl;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
if(transform_type == fft_transform_type_real_forward
|
|
|| transform_type == fft_transform_type_real_inverse)
|
|
{
|
|
bool invalid = length.back() > 1 && (istride.back() != 1 || ostride.back() != 1);
|
|
if(invalid)
|
|
{
|
|
// In-place real/complex transforms require unit strides.
|
|
if(verbose)
|
|
{
|
|
std::cout << "istride.back(): " << istride.back()
|
|
<< " ostride.back(): " << ostride.back()
|
|
<< " must be unitary for in-place real/complex transforms: "
|
|
"skipping test"
|
|
<< std::endl;
|
|
}
|
|
return false;
|
|
}
|
|
const auto& real_strides
|
|
= transform_type == fft_transform_type_real_forward ? istride : ostride;
|
|
const auto& hermitian_strides
|
|
= transform_type == fft_transform_type_real_forward ? ostride : istride;
|
|
const auto& real_dist
|
|
= transform_type == fft_transform_type_real_forward ? idist : odist;
|
|
const auto& hermitian_dist
|
|
= transform_type == fft_transform_type_real_forward ? odist : idist;
|
|
for(size_t dim = 0; dim < stridesize - 1; dim++)
|
|
{
|
|
if(length[dim] == 1)
|
|
continue;
|
|
invalid |= real_strides[dim] != 2 * hermitian_strides[dim];
|
|
}
|
|
if(nbatch > 1)
|
|
invalid |= real_dist != 2 * hermitian_dist;
|
|
if(invalid)
|
|
{
|
|
if(verbose)
|
|
{
|
|
std::cout << "Inconsistency detected in strides/distances for in-place "
|
|
"real/complex transforms; skipped\n";
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if((itype == fft_array_type_complex_interleaved
|
|
&& otype == fft_array_type_complex_planar)
|
|
|| (itype == fft_array_type_complex_planar
|
|
&& otype == fft_array_type_complex_interleaved))
|
|
{
|
|
if(verbose)
|
|
{
|
|
std::cout << "In-place c2c transforms require identical io types; skipped.\n";
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Check offsets
|
|
switch(transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
case fft_transform_type_complex_inverse:
|
|
for(unsigned int i = 0; i < nibuffer(); ++i)
|
|
{
|
|
if(ioffset[i] != ooffset[i])
|
|
return false;
|
|
}
|
|
break;
|
|
case fft_transform_type_real_forward:
|
|
if(ioffset[0] != 2 * ooffset[0])
|
|
return false;
|
|
break;
|
|
case fft_transform_type_real_inverse:
|
|
if(2 * ioffset[0] != ooffset[0])
|
|
return false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if(!check_iotypes())
|
|
return false;
|
|
|
|
// we can only check output strides on out-of-place
|
|
// transforms, since we need to initialize output to a known
|
|
// pattern
|
|
if(placement == fft_placement_inplace && check_output_strides)
|
|
return false;
|
|
|
|
// Check input and output strides
|
|
if(valid_length_stride_batch_dist(ilength(), istride, nbatch, idist, verbose) != true)
|
|
{
|
|
if(verbose)
|
|
std::cout << "Invalid input data format.\n";
|
|
return false;
|
|
}
|
|
if(!(ilength() == olength() && istride == ostride && idist == odist))
|
|
{
|
|
// Only check if different
|
|
if(valid_length_stride_batch_dist(olength(), ostride, nbatch, odist, verbose) != true)
|
|
{
|
|
if(verbose)
|
|
std::cout << "Invalid output data format.\n";
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// The parameters are valid.
|
|
return true;
|
|
}
|
|
|
|
// TODO: temporary workaround awaiting robust support for
|
|
// 64-bit indexing in rocfft kernels.
|
|
bool may_need_64bit_indexing() const
|
|
{
|
|
for(auto io : {fft_io::fft_io_in, fft_io::fft_io_out})
|
|
{
|
|
const auto& io_stride = io == fft_io::fft_io_in ? istride : ostride;
|
|
const auto& io_dist = io == fft_io::fft_io_in ? idist : odist;
|
|
const auto& io_array_type = io == fft_io::fft_io_in ? itype : otype;
|
|
const auto& io_offset = io == fft_io::fft_io_in ? ioffset : ooffset;
|
|
const auto io_length = io == fft_io::fft_io_in ? ilength() : olength();
|
|
const auto max_offset
|
|
= io_offset.empty() ? 0 : *std::max_element(io_offset.begin(), io_offset.end());
|
|
// Hermitian interleaved data may be re-interpreted as real data internally.
|
|
if((max_offset + compute_ptrdiff(io_length, io_stride, nbatch, io_dist))
|
|
* (io_array_type == fft_array_type_hermitian_interleaved ? 2 : 1)
|
|
> static_cast<size_t>(UINT32_MAX) + 1)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Fill in any missing parameters.
|
|
void validate()
|
|
{
|
|
set_iotypes();
|
|
compute_istride();
|
|
compute_ostride();
|
|
set_idist();
|
|
set_odist();
|
|
compute_isize();
|
|
compute_osize();
|
|
|
|
validate_fields();
|
|
}
|
|
|
|
// validate that the bricks in the fields have positive volume
|
|
// (i.e. upper index is above lower index). also check that they
|
|
// have the right number of dimensions for the fields
|
|
void validate_brick_volume() const
|
|
{
|
|
// row-major lengths including batch (i.e. batch is at the front)
|
|
std::vector<size_t> length_with_batch{nbatch};
|
|
std::copy(length.begin(), length.end(), std::back_inserter(length_with_batch));
|
|
|
|
auto validate_field = [&](const fft_field& f) {
|
|
for(const auto& b : f.bricks)
|
|
{
|
|
// bricks must have same dim as FFT, including batch
|
|
if(b.lower.size() != length.size() + 1 || b.upper.size() != length.size() + 1
|
|
|| b.stride.size() != length.size() + 1)
|
|
throw std::runtime_error(
|
|
"brick dimension does not match FFT + batch dimension");
|
|
|
|
// ensure lower < upper, and that both fit in the FFT + batch dims
|
|
if(!std::lexicographical_compare(
|
|
b.lower.begin(), b.lower.end(), b.upper.begin(), b.upper.end()))
|
|
throw std::runtime_error("brick lower index is not less than upper index");
|
|
|
|
if(!std::lexicographical_compare(b.lower.begin(),
|
|
b.lower.end(),
|
|
length_with_batch.begin(),
|
|
length_with_batch.end()))
|
|
throw std::runtime_error(
|
|
"brick lower index is not less than FFT + batch length");
|
|
|
|
if(!std::lexicographical_compare(b.upper.begin(),
|
|
b.upper.end(),
|
|
length_with_batch.begin(),
|
|
length_with_batch.end())
|
|
&& b.upper != length_with_batch)
|
|
throw std::runtime_error("brick upper index is not <= FFT + batch length");
|
|
}
|
|
};
|
|
|
|
for(const auto& ifield : ifields)
|
|
validate_field(ifield);
|
|
for(const auto& ofield : ofields)
|
|
validate_field(ofield);
|
|
}
|
|
|
|
virtual void validate_fields() const
|
|
{
|
|
if(multiGPU > 1)
|
|
throw std::runtime_error("library-decomposed multi-GPU is unsupported");
|
|
}
|
|
|
|
// Column-major getters:
|
|
std::vector<size_t> length_cm() const
|
|
{
|
|
auto length_cm = length;
|
|
std::reverse(std::begin(length_cm), std::end(length_cm));
|
|
return length_cm;
|
|
}
|
|
std::vector<size_t> ilength_cm() const
|
|
{
|
|
auto ilength_cm = ilength();
|
|
std::reverse(std::begin(ilength_cm), std::end(ilength_cm));
|
|
return ilength_cm;
|
|
}
|
|
std::vector<size_t> olength_cm() const
|
|
{
|
|
auto olength_cm = olength();
|
|
std::reverse(std::begin(olength_cm), std::end(olength_cm));
|
|
return olength_cm;
|
|
}
|
|
std::vector<size_t> istride_cm() const
|
|
{
|
|
auto istride_cm = istride;
|
|
std::reverse(std::begin(istride_cm), std::end(istride_cm));
|
|
return istride_cm;
|
|
}
|
|
std::vector<size_t> ostride_cm() const
|
|
{
|
|
auto ostride_cm = ostride;
|
|
std::reverse(std::begin(ostride_cm), std::end(ostride_cm));
|
|
return ostride_cm;
|
|
}
|
|
bool is_interleaved() const
|
|
{
|
|
return array_type_is_interleaved(itype) || array_type_is_interleaved(otype);
|
|
}
|
|
bool is_planar() const
|
|
{
|
|
return array_type_is_planar(itype) || array_type_is_planar(otype);
|
|
}
|
|
bool is_real() const
|
|
{
|
|
return (itype == fft_array_type_real || otype == fft_array_type_real);
|
|
}
|
|
bool is_callback() const
|
|
{
|
|
return run_callbacks != fft_callback_type_none;
|
|
}
|
|
// checks if the parameters are consistent with a "default" data layout (considering strides and distances)
|
|
bool is_using_default_layout() const
|
|
{
|
|
static_assert(std::is_same_v<decltype(ioffset), decltype(ooffset)>);
|
|
auto is_zero = [](const decltype(ioffset)::value_type& i) { return i == 0; };
|
|
return std::all_of(ioffset.begin(), ioffset.end(), is_zero)
|
|
&& std::all_of(ooffset.begin(), ooffset.end(), is_zero)
|
|
&& istride == default_strides(transform_type, placement, fft_io::fft_io_in, length)
|
|
&& ostride == default_strides(transform_type, placement, fft_io::fft_io_out, length)
|
|
&& idist
|
|
== default_distance(
|
|
transform_type, placement, fft_io::fft_io_in, length, nbatch)
|
|
&& odist
|
|
== default_distance(
|
|
transform_type, placement, fft_io::fft_io_out, length, nbatch);
|
|
}
|
|
|
|
// Given a data type and dimensions, fill the buffer, imposing Hermitian symmetry if necessary.
|
|
template <typename Tbuff>
|
|
inline void compute_input(std::vector<Tbuff>& input)
|
|
{
|
|
auto deviceProp = get_curr_device_prop();
|
|
|
|
std::vector<size_t> field_lower(dim());
|
|
auto contiguous_stride = compute_stride(ilength());
|
|
auto contiguous_dist = compute_idist();
|
|
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
set_input<Tbuff, rocfft_fp16>(input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength(),
|
|
istride,
|
|
ioffset,
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
field_lower,
|
|
0,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
case fft_precision_double:
|
|
set_input<Tbuff, double>(input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength(),
|
|
istride,
|
|
ioffset,
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
field_lower,
|
|
0,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
case fft_precision_single:
|
|
set_input<Tbuff, float>(input,
|
|
igen,
|
|
itype,
|
|
length,
|
|
ilength(),
|
|
istride,
|
|
ioffset,
|
|
idist,
|
|
nbatch,
|
|
deviceProp,
|
|
field_lower,
|
|
0,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
}
|
|
}
|
|
|
|
template <typename Tstream = std::ostream>
|
|
void print_ibuffer(const std::vector<hostbuf>& buf, Tstream& stream = std::cout) const
|
|
{
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_complex<rocfft_fp16>> s;
|
|
s.print_buffer_half(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<rocfft_complex<float>> s;
|
|
s.print_buffer_single(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
buffer_printer<rocfft_complex<double>> s;
|
|
s.print_buffer_double(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_fp16> s;
|
|
s.print_buffer_half(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<float> s;
|
|
s.print_buffer_single(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
buffer_printer<double> s;
|
|
s.print_buffer_double(buf, ilength(), istride, nbatch, idist, ioffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
throw std::runtime_error("Invalid itype in print_ibuffer");
|
|
}
|
|
}
|
|
|
|
template <typename Tstream = std::ostream>
|
|
void print_obuffer(const std::vector<hostbuf>& buf, Tstream& stream = std::cout) const
|
|
{
|
|
switch(otype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_complex<rocfft_fp16>> s;
|
|
s.print_buffer_half(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<rocfft_complex<float>> s;
|
|
s.print_buffer_single(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
buffer_printer<rocfft_complex<double>> s;
|
|
s.print_buffer_double(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_fp16> s;
|
|
s.print_buffer_half(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<float> s;
|
|
s.print_buffer_single(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
buffer_printer<double> s;
|
|
s.print_buffer_double(buf, olength(), ostride, nbatch, odist, ooffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
default:
|
|
throw std::runtime_error("Invalid itype in print_obuffer");
|
|
}
|
|
}
|
|
|
|
void print_ibuffer_flat(const std::vector<hostbuf>& buf) const
|
|
{
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_complex<rocfft_fp16>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<rocfft_complex<float>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
buffer_printer<rocfft_complex<double>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_fp16> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<float> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
buffer_printer<double> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid itype in print_ibuffer_flat");
|
|
}
|
|
}
|
|
}
|
|
|
|
void print_obuffer_flat(const std::vector<hostbuf>& buf) const
|
|
{
|
|
switch(otype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_complex<rocfft_fp16>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<rocfft_complex<float>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
buffer_printer<rocfft_complex<double>> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
case fft_array_type_real:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
buffer_printer<rocfft_fp16> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
buffer_printer<float> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
|
|
case fft_precision_double:
|
|
{
|
|
buffer_printer<double> s;
|
|
s.print_buffer_flat(buf, osize, ooffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid itype in print_ibuffer_flat");
|
|
}
|
|
}
|
|
}
|
|
|
|
// A function pointer callback is expressed as a pair of device
|
|
// function pointer + device function data.
|
|
//
|
|
// Load and store callbacks are provided as vectors of those
|
|
// pointers, as we need a separate function+data for each device
|
|
// being loaded from or stored to.
|
|
virtual fft_status set_funcptr_callbacks(std::vector<void*>* load_cb_func,
|
|
std::vector<void*>* load_cb_data,
|
|
std::vector<void*>* store_cb_func,
|
|
std::vector<void*>* store_cb_data,
|
|
size_t load_cb_shared_mem_bytes,
|
|
size_t store_cb_shared_mem_bytes)
|
|
{
|
|
return fft_status_success;
|
|
}
|
|
|
|
virtual fft_status execute(void** in, void** out)
|
|
{
|
|
return fft_status_success;
|
|
}
|
|
|
|
std::vector<size_t> footprint_on_devices_for(fft_io io) const
|
|
{
|
|
std::vector<size_t> sizes(rocfft_scoped_device::device_count());
|
|
const auto& iofields = io == fft_io::fft_io_in ? ifields : ofields;
|
|
const auto iobuffer_size
|
|
= io == fft_io::fft_io_in ? sum(ibuffer_sizes()) : sum(obuffer_sizes());
|
|
|
|
// If this is library-decomposed multi-GPU, only the library
|
|
// can really say what the footprint will be. Estimate it
|
|
// here by assuming the input/output size will be evenly
|
|
// divided across all of the devices.
|
|
if(multiGPU > 1)
|
|
{
|
|
// We will use the first N devices for library-decomposed
|
|
for(size_t device = 0; device < multiGPU; ++device)
|
|
{
|
|
sizes.at(device) += DivRoundingUp(iobuffer_size, multiGPU);
|
|
}
|
|
return sizes;
|
|
}
|
|
|
|
int currentDevice = hipInvalidDeviceId;
|
|
if(hipGetDevice(¤tDevice) != hipSuccess)
|
|
throw std::runtime_error("hipGetDevice failed");
|
|
|
|
// add sizes for field if specified, otherwise assume
|
|
// single-device input/output buffer on current device
|
|
if(iofields.empty())
|
|
{
|
|
sizes.at(currentDevice) += iobuffer_size;
|
|
}
|
|
else
|
|
{
|
|
// add footprint of each brick to its device
|
|
for(const auto& field : iofields)
|
|
{
|
|
for(const auto& brick : field.bricks)
|
|
{
|
|
sizes.at(brick.device) += compute_ptrdiff(brick.length(), brick.stride);
|
|
}
|
|
}
|
|
}
|
|
return sizes;
|
|
}
|
|
|
|
// return the per-device memory footprint of just the input/output data
|
|
std::vector<size_t> io_vram_footprint() const
|
|
{
|
|
std::vector<size_t> sizes(rocfft_scoped_device::device_count());
|
|
const auto input_footprints = footprint_on_devices_for(fft_io::fft_io_in);
|
|
const auto output_footprints = footprint_on_devices_for(fft_io::fft_io_out);
|
|
|
|
for(size_t i = 0; i < sizes.size(); ++i)
|
|
{
|
|
if(placement == fft_placement_inplace)
|
|
{
|
|
// inplace means we need the maximum of input and output for each device
|
|
sizes.at(i) = std::max(input_footprints.at(i), output_footprints.at(i));
|
|
}
|
|
else
|
|
{
|
|
// out of place means input and output are separate
|
|
sizes.at(i) = input_footprints.at(i) + output_footprints.at(i);
|
|
}
|
|
}
|
|
return sizes;
|
|
}
|
|
|
|
// return the full per-device memory footprint of the FFT including
|
|
// any work buffers required by the FFT library
|
|
virtual std::vector<size_t> vram_footprint()
|
|
{
|
|
return io_vram_footprint();
|
|
}
|
|
|
|
// Specific exception type for work buffer allocation failure.
|
|
// Tests that hit this can't fit on the GPU and should be skipped.
|
|
struct work_buffer_alloc_failure : public hip_runtime_error
|
|
{
|
|
const size_t attempted_size;
|
|
work_buffer_alloc_failure(const std::string& s,
|
|
size_t _attempted_size = 0,
|
|
hipError_t hip_status = hipErrorUnknown)
|
|
: hip_runtime_error(s, hip_status)
|
|
, attempted_size(_attempted_size)
|
|
{
|
|
}
|
|
};
|
|
|
|
// Specific exception type for unimplemented feature(s).
|
|
struct unimplemented_exception : public std::runtime_error
|
|
{
|
|
unimplemented_exception(const std::string& s)
|
|
: std::runtime_error(s)
|
|
{
|
|
}
|
|
};
|
|
|
|
virtual fft_status create_plan()
|
|
{
|
|
return fft_status_success;
|
|
}
|
|
|
|
// Change a forward transform to it's inverse
|
|
void inverse_from_forward(fft_params& params_forward)
|
|
{
|
|
switch(params_forward.transform_type)
|
|
{
|
|
case fft_transform_type_complex_forward:
|
|
transform_type = fft_transform_type_complex_inverse;
|
|
break;
|
|
case fft_transform_type_real_forward:
|
|
transform_type = fft_transform_type_real_inverse;
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Transform type not forward.");
|
|
}
|
|
|
|
length = params_forward.length;
|
|
istride = params_forward.ostride;
|
|
ostride = params_forward.istride;
|
|
nbatch = params_forward.nbatch;
|
|
precision = params_forward.precision;
|
|
placement = params_forward.placement;
|
|
idist = params_forward.odist;
|
|
odist = params_forward.idist;
|
|
itype = params_forward.otype;
|
|
otype = params_forward.itype;
|
|
ioffset = params_forward.ooffset;
|
|
ooffset = params_forward.ioffset;
|
|
auto_allocate = params_forward.auto_allocate;
|
|
|
|
run_callbacks = params_forward.run_callbacks;
|
|
multiGPU = params_forward.multiGPU;
|
|
|
|
check_output_strides = params_forward.check_output_strides;
|
|
|
|
scale_factor = 1 / params_forward.scale_factor;
|
|
}
|
|
|
|
/**
|
|
* @brief initialize the I/O device buffers as required for multi-device
|
|
* transforms described by this object. Device buffers are determined (created
|
|
* and allocated, if needed), and input buffers are data-initialized using the
|
|
* provided host-residing (resp. device-residing) reference data for the
|
|
* `rocfft_params` (resp. `hipfft_params`) derived class.
|
|
*
|
|
* TODO: unify data-initialization logic across derived classes and make all arguments
|
|
* relevant in all cases (remove ignore ones).
|
|
*
|
|
* @param[in] input_data_host vector of (at least) one host buffer containing the
|
|
* input-initialization data to be considered for data-initialization of the
|
|
* device input buffers (this is unused by `hipfft_params` objects). If used,
|
|
* the data layout of `input_data_host[0]` must be consistent with `cpu_params.istride`,
|
|
* `cpu_params.idist`, etc. wherein `cpu_params = this->make_params_for_reference_cpu()`.
|
|
* @param[in] input_data_gpu vector of (at least) one device buffer containing the
|
|
* input-initialization data to be considered for data-initialization of the
|
|
* device input buffers (this is unused by `rocfft_params` objects). If used,
|
|
* the data layout of `input_data_gpu[0]` must be consistent with `this->istride`,
|
|
* `this->idist`, etc.
|
|
* @param[out] mgpu_ibuffers vector of raw pointers to input device allocations as
|
|
* needed for the multi-device transform that this object describes. The content
|
|
* of these device allocations is initialized with the (transposed) content of
|
|
* `input_data_host` (resp. `input_data_gpu`) by `rocfft_params` (resp. `hipfft_params`)
|
|
* objects. Upon return, the data layout in these device allocations is
|
|
* - as described by the corresponding input brick's for `rocfft_params` objects;
|
|
* - as implicitly-defined for `hipfft_params` objects.
|
|
* @param[out] mgpu_obuffers vector of raw pointers to output device allocations as
|
|
* needed for the multi-device transform that this object describes.
|
|
*/
|
|
virtual void multi_gpu_prepare(const std::vector<hostbuf>& input_data_host,
|
|
const std::vector<gpubuf>& input_data_gpu,
|
|
std::vector<void*>& mgpu_ibuffers,
|
|
std::vector<void*>& mgpu_obuffers)
|
|
{
|
|
}
|
|
|
|
/**
|
|
* @brief gather the results of a multi-device transform to a unique host-residing
|
|
* buffer for `rocfft_params` objects (resp. device-residing buffer for
|
|
* `hipfft_params` objects).
|
|
*
|
|
* TODO: unify data-gathering logic across derived classes and make all arguments
|
|
* relevant in all cases (remove ignore ones).
|
|
*
|
|
* @param[out] gathered_results_host vector of (at least) one host buffer (ignored by
|
|
* `hipfft_params` objects). If not ignored, `gathered_results_host[0]` contains the
|
|
* gathered output data of the multi-device transform upon return.
|
|
* @param[out] gathered_results_device vector of (at least) one device buffer (ignored
|
|
* by `rocfft_params` objects). If not ignored, `gathered_results_device[0]` contains
|
|
* the gathered output data of the multi-device transform upon return.
|
|
* @param[in] mgpu_obuffers vector of raw pointers to output device allocations
|
|
* considered by the (previously-executed) multi-device transform described by this
|
|
* object.
|
|
*
|
|
* Note 1: whichever gathering buffer that is considered (`gathered_results_host[0]`
|
|
* or `gathered_results_device[0]`) must be large enough for containing the
|
|
* gathered results when passed to this function: this function does NOT resize it.
|
|
* Note 2: the data layout in the considered gathered results is defined
|
|
* by the calling object's output data layout parameter, i.e., defined by
|
|
* `this->ostride`, `this->odist`, `this->otype`, etc.
|
|
*/
|
|
virtual void multi_gpu_finalize(std::vector<hostbuf>& gathered_results_host,
|
|
std::vector<gpubuf>& gathered_results_device,
|
|
std::vector<void*>& mgpu_obuffers)
|
|
{
|
|
}
|
|
|
|
/**
|
|
* @brief create one input (resp. output) field with as many bricks
|
|
* as desired along every field dimension for the current object.
|
|
* The device assignment of the bricks is cycling though the total number
|
|
* of available devices (i.e., `gpusperrank * num_ranks`) starting from
|
|
* the `start_global_dev_idx` (global) device index.
|
|
* NOTE: The global device index `g` in `[0, gpusperrank * mp_ranks(`
|
|
* corresponds to device of local ID `g / mp_ranks` on process of rank
|
|
* `g % mp_ranks`. This prioritize distributing layout across all processes.
|
|
*
|
|
* @tparam io enum flag specifying whether the input (`io == fft_io::fft_io_in`)
|
|
* or output (`io == fft_io::fft_io_in`) field is being considered.
|
|
*
|
|
* @param[in] gpusperrank number of GPU devices available to each process
|
|
* @param[in] brick_count_along desired (strictly positive) numbers of
|
|
* bricks per dimension: `brick_count_along[i]` if the number of bricks
|
|
* desired along the `i`-th field dimension.
|
|
* @param[in] num_ranks number of processes used in the parallel computer
|
|
* (default value is 1)
|
|
* @param[in] start_global_dev_idx first (global) device ID to be considered
|
|
* in the cyclic assignment (default value is 0)
|
|
*
|
|
* NOTES:
|
|
* - data layouts within bricks are set to be compact, i.e., contiguous for
|
|
* bricks in complex domain, padded in real domain for in-place operations
|
|
* (only if the fastest dimension is not divided itself, i.e., if
|
|
* brick_count_along.last() == 1)
|
|
* - if more bricks than available devices are requested, some bricks may be
|
|
* assigned to the same device.
|
|
*/
|
|
template <fft_io io>
|
|
void distribute_field(int gpusperrank,
|
|
const std::vector<unsigned int>& brick_count_along,
|
|
int num_ranks = 1,
|
|
int start_global_dev_idx = 0)
|
|
{
|
|
static_assert(io == fft_io::fft_io_in || io == fft_io::fft_io_out);
|
|
|
|
auto& iofields = io == fft_io::fft_io_in ? ifields : ofields;
|
|
auto iofield_length = io == fft_io::fft_io_in ? ilength() : olength();
|
|
iofield_length.insert(iofield_length.begin(), nbatch);
|
|
const auto field_size = iofield_length.size(); // --> field_size > 0
|
|
// arg validation
|
|
if(brick_count_along.size() != field_size)
|
|
throw std::runtime_error(
|
|
"fft_params::distribute_field inconsistent size between desired number of bricks "
|
|
"per dimension and number of dimensions");
|
|
if(std::any_of(
|
|
brick_count_along.begin(),
|
|
brick_count_along.end(),
|
|
[](const auto& brick_count_along_dim) { return brick_count_along_dim == 0; }))
|
|
throw std::invalid_argument("brick_count_per_dim must not be 0.");
|
|
if(gpusperrank < 1 || num_ranks < 1)
|
|
throw std::invalid_argument(
|
|
"fft_params::distribute_field requires strictly positive number of processes and "
|
|
"number of available device(s) per process.");
|
|
|
|
const auto total_bricks = product(brick_count_along.begin(), brick_count_along.end());
|
|
|
|
if(total_bricks == 1 && mp_lib == fft_mp_lib_none && num_ranks == 1
|
|
&& start_global_dev_idx == 0)
|
|
{
|
|
// Specifying a unique field with a lone brick on the current device may
|
|
// be omitted in favor of plan's data layout parameters: test that by not
|
|
// using a lone-brick field sometimes, in such cases.
|
|
const auto stable_hash_str
|
|
= token() + (io == fft_io::fft_io_in ? "_input_field" : "_output_field");
|
|
if(std::hash<std::string>()(stable_hash_str) % 2 == 1)
|
|
return;
|
|
}
|
|
|
|
struct length_division
|
|
{
|
|
size_t min_length;
|
|
size_t remainder;
|
|
};
|
|
std::vector<length_division> field_division_along(field_size);
|
|
for(auto field_dim = field_size; field_dim-- > 0;)
|
|
{
|
|
field_division_along[field_dim].min_length
|
|
= iofield_length[field_dim] / brick_count_along[field_dim];
|
|
field_division_along[field_dim].remainder
|
|
= iofield_length[field_dim] % brick_count_along[field_dim];
|
|
}
|
|
|
|
// make ONE field with as many bricks as required along every field dimension
|
|
iofields.resize(1);
|
|
auto& iofield = iofields[0];
|
|
auto& iobricks = iofield.bricks;
|
|
// Create the required number of bricks
|
|
iobricks.resize(total_bricks);
|
|
// define them
|
|
for(size_t b_idx = 0; b_idx < total_bricks; b_idx++)
|
|
{
|
|
auto& iobrick = iobricks[b_idx];
|
|
std::vector<size_t> brick_dim_idx(field_size, 0);
|
|
iobrick.lower.resize(field_size);
|
|
iobrick.upper.resize(field_size);
|
|
iobrick.stride.resize(field_size);
|
|
auto tmp_idx = b_idx;
|
|
for(auto field_dim = field_size; field_dim-- > 0;
|
|
tmp_idx /= brick_count_along[field_dim])
|
|
{
|
|
const auto brick_idx_along_dim = tmp_idx % brick_count_along[field_dim];
|
|
const auto brick_length_along_dim
|
|
= field_division_along[field_dim].min_length
|
|
+ (brick_idx_along_dim < field_division_along[field_dim].remainder ? 1 : 0);
|
|
iobrick.lower[field_dim]
|
|
= brick_idx_along_dim * field_division_along[field_dim].min_length
|
|
+ std::min(brick_idx_along_dim, field_division_along[field_dim].remainder);
|
|
iobrick.upper[field_dim] = iobrick.lower[field_dim] + brick_length_along_dim;
|
|
if(field_dim == field_size - 1)
|
|
iobrick.stride[field_dim] = 1; // contiguous along fastest dimension
|
|
else
|
|
{
|
|
auto stride_multiplier
|
|
= iobrick.upper[field_dim + 1] - iobrick.lower[field_dim + 1];
|
|
if(field_dim == field_size - 2 && is_real()
|
|
&& placement == fft_placement_inplace
|
|
&& (is_forward() ^ (io == fft_io::fft_io_out)) /* <-- brick in real domain */
|
|
&& brick_count_along[field_size - 1] == 1)
|
|
{
|
|
// real in-place case with fastest dimension that is NOT divided
|
|
// --> most likely use case is *with* padding
|
|
stride_multiplier = 2 * (stride_multiplier / 2 + 1);
|
|
}
|
|
iobrick.stride[field_dim] = iobrick.stride[field_dim + 1] * stride_multiplier;
|
|
}
|
|
}
|
|
|
|
iobrick.rank = (start_global_dev_idx + b_idx) % num_ranks;
|
|
iobrick.device = ((start_global_dev_idx + b_idx) / num_ranks) % gpusperrank;
|
|
}
|
|
}
|
|
|
|
// Apply load operations specified by this struct to the host-side
|
|
// input before we pass it to the reference FFT
|
|
void apply_host_load_ops(std::vector<hostbuf>& input) const
|
|
{
|
|
// Currently no load operations can be specified
|
|
}
|
|
|
|
// Apply store operations specified by this struct to the host-side
|
|
// output after we get it from the reference FFT
|
|
void apply_host_store_ops(std::vector<hostbuf>& output) const
|
|
{
|
|
// Store ops like result scaling are only supported on AMD
|
|
// backend, and CUDA implements some conflicting
|
|
// half-precision operators that prevent result scaling from
|
|
// compiling
|
|
#ifdef __HIP_PLATFORM_AMD__
|
|
// Don't bother iterating over the data if we don't have to
|
|
if(scale_factor == 1.0)
|
|
return;
|
|
#ifdef _OPENMP
|
|
auto partition_count = compute_partition_count(output.front().size());
|
|
#endif
|
|
|
|
switch(otype)
|
|
{
|
|
// Reference FFT always works with complex-interleaved data even if
|
|
// params specifies planar
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
const size_t elem_size = sizeof(rocfft_complex<rocfft_fp16>);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin
|
|
= reinterpret_cast<rocfft_complex<rocfft_fp16>*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
const size_t elem_size = sizeof(rocfft_complex<float>);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin = reinterpret_cast<rocfft_complex<float>*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
const size_t elem_size = sizeof(rocfft_complex<double>);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin
|
|
= reinterpret_cast<rocfft_complex<double>*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
case fft_array_type_real:
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
{
|
|
const size_t elem_size = sizeof(rocfft_fp16);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin = reinterpret_cast<rocfft_fp16*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
case fft_precision_single:
|
|
{
|
|
const size_t elem_size = sizeof(float);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin = reinterpret_cast<float*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
case fft_precision_double:
|
|
{
|
|
const size_t elem_size = sizeof(double);
|
|
const size_t num_elems = output.front().size() / elem_size;
|
|
|
|
auto output_begin = reinterpret_cast<double*>(output.front().data());
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partition_count)
|
|
#endif
|
|
for(size_t i = 0; i < num_elems; ++i)
|
|
{
|
|
auto& element = output_begin[i];
|
|
if(scale_factor != 1.0)
|
|
element = element * scale_factor;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
// this is FFTW data which should always be interleaved (if complex)
|
|
abort();
|
|
}
|
|
#endif
|
|
}
|
|
|
|
fft_params make_params_for_reference_cpu() const
|
|
{
|
|
fft_params ret;
|
|
ret.length = length;
|
|
ret.precision = precision;
|
|
ret.placement = fft_placement_notinplace;
|
|
ret.transform_type = transform_type;
|
|
ret.nbatch = nbatch;
|
|
ret.run_callbacks = run_callbacks;
|
|
ret.scale_factor = scale_factor;
|
|
ret.itype = is_real() ? (is_forward() ? fft_array_type_real
|
|
: fft_array_type_hermitian_interleaved)
|
|
: fft_array_type_complex_interleaved;
|
|
ret.otype = is_real() ? (is_inverse() ? fft_array_type_real
|
|
: fft_array_type_hermitian_interleaved)
|
|
: fft_array_type_complex_interleaved;
|
|
ret.istride
|
|
= default_strides(transform_type, fft_placement_notinplace, fft_io::fft_io_in, length);
|
|
ret.ostride
|
|
= default_strides(transform_type, fft_placement_notinplace, fft_io::fft_io_out, length);
|
|
ret.idist = default_distance(
|
|
transform_type, fft_placement_notinplace, fft_io::fft_io_in, length, nbatch);
|
|
ret.odist = default_distance(
|
|
transform_type, fft_placement_notinplace, fft_io::fft_io_out, length, nbatch);
|
|
|
|
ret.validate();
|
|
|
|
// other ret's members should be irrelevant for cpu reference calculations
|
|
// (default values)
|
|
return ret;
|
|
}
|
|
|
|
int get_process_rank() const
|
|
{
|
|
int process_rank = -1; // invalid initialization
|
|
if(mp_lib == fft_mp_lib_mpi)
|
|
{
|
|
#ifdef ROCFFT_MPI_ENABLE
|
|
if(!mp_comm)
|
|
throw std::runtime_error("Multi-process communicator is not defined");
|
|
auto ret = MPI_Comm_rank(*static_cast<MPI_Comm*>(mp_comm), &process_rank);
|
|
if(ret != MPI_SUCCESS || process_rank < 0)
|
|
throw std::runtime_error("Rank of current process couldn't be set");
|
|
#else
|
|
throw std::runtime_error("MPI is not enabled");
|
|
#endif
|
|
}
|
|
else
|
|
{
|
|
process_rank = 0;
|
|
}
|
|
return process_rank;
|
|
}
|
|
};
|
|
|
|
// Used for CLI11 parsing of multi-process library enum
|
|
static bool lexical_cast(const std::string& word, fft_params::fft_mp_lib& mp_lib)
|
|
{
|
|
if(word == "none")
|
|
mp_lib = fft_params::fft_mp_lib_none;
|
|
else if(word == "mpi")
|
|
mp_lib = fft_params::fft_mp_lib_mpi;
|
|
else
|
|
throw std::runtime_error("Invalid multi-process library specified");
|
|
return true;
|
|
}
|
|
|
|
// Used for CLI11 parsing of callbacks enum
|
|
static bool lexical_cast(const std::string& word, fft_callback_type& cbtype)
|
|
{
|
|
if(word == "none")
|
|
cbtype = fft_callback_type_none;
|
|
else if(word == "funcptr")
|
|
cbtype = fft_callback_type_funcptr;
|
|
else if(word == "jit")
|
|
cbtype = fft_callback_type_jit;
|
|
else
|
|
throw std::runtime_error("Invalid callback type specified");
|
|
return true;
|
|
}
|
|
|
|
// This is used with CLI11 so that the user can type an integer on the
|
|
// command line and we store into an enum variable
|
|
template <typename _Elem, typename _Traits>
|
|
std::basic_istream<_Elem, _Traits>& operator>>(std::basic_istream<_Elem, _Traits>& stream,
|
|
fft_array_type& atype)
|
|
{
|
|
unsigned tmp;
|
|
stream >> tmp;
|
|
atype = fft_array_type(tmp);
|
|
return stream;
|
|
}
|
|
|
|
// Similarly for transform type
|
|
template <typename _Elem, typename _Traits>
|
|
std::basic_istream<_Elem, _Traits>& operator>>(std::basic_istream<_Elem, _Traits>& stream,
|
|
fft_transform_type& ttype)
|
|
{
|
|
unsigned tmp;
|
|
stream >> tmp;
|
|
ttype = fft_transform_type(tmp);
|
|
return stream;
|
|
}
|
|
|
|
// Returns pairs of startindex, endindex, for 1D, 2D, 3D lengths
|
|
template <typename T1>
|
|
std::vector<std::pair<T1, T1>> partition_colmajor(const T1& length)
|
|
{
|
|
return partition_base(length, compute_partition_count(length));
|
|
}
|
|
|
|
// Partition on the rightmost part of the tuple, for col-major indexing
|
|
template <typename T1>
|
|
std::vector<std::pair<std::tuple<T1, T1>, std::tuple<T1, T1>>>
|
|
partition_colmajor(const std::tuple<T1, T1>& length)
|
|
{
|
|
auto partitions = partition_base(std::get<1>(length), compute_partition_count(length));
|
|
std::vector<std::pair<std::tuple<T1, T1>, std::tuple<T1, T1>>> ret(partitions.size());
|
|
for(size_t i = 0; i < partitions.size(); ++i)
|
|
{
|
|
std::get<1>(ret[i].first) = partitions[i].first;
|
|
std::get<0>(ret[i].first) = 0;
|
|
std::get<1>(ret[i].second) = partitions[i].second;
|
|
std::get<0>(ret[i].second) = std::get<0>(length);
|
|
}
|
|
return ret;
|
|
}
|
|
template <typename T1>
|
|
std::vector<std::pair<std::tuple<T1, T1, T1>, std::tuple<T1, T1, T1>>>
|
|
partition_colmajor(const std::tuple<T1, T1, T1>& length)
|
|
{
|
|
auto partitions = partition_base(std::get<2>(length), compute_partition_count(length));
|
|
std::vector<std::pair<std::tuple<T1, T1, T1>, std::tuple<T1, T1, T1>>> ret(partitions.size());
|
|
for(size_t i = 0; i < partitions.size(); ++i)
|
|
{
|
|
std::get<2>(ret[i].first) = partitions[i].first;
|
|
std::get<1>(ret[i].first) = 0;
|
|
std::get<0>(ret[i].first) = 0;
|
|
std::get<2>(ret[i].second) = partitions[i].second;
|
|
std::get<1>(ret[i].second) = std::get<1>(length);
|
|
std::get<0>(ret[i].second) = std::get<0>(length);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// Copy data of dimensions length with strides istride and length idist between batches to
|
|
// a buffer with strides ostride and length odist between batches. The input and output
|
|
// types are identical.
|
|
template <typename Tval, typename Tint1, typename Tint2, typename Tint3>
|
|
inline void copy_buffers_1to1(const Tval* input,
|
|
Tval* output,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset)
|
|
{
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partitions.size())
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
output[odx + ooffset[0]] = input[idx + ioffset[0]];
|
|
} while(increment_rowmajor(index, length));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Copy data of dimensions length with strides istride and length idist between batches to
|
|
// a buffer with strides ostride and length odist between batches. The input type is
|
|
// planar and the output type is complex interleaved.
|
|
template <typename Tval, typename Tint1, typename Tint2, typename Tint3>
|
|
inline void copy_buffers_2to1(const Tval* input0,
|
|
const Tval* input1,
|
|
rocfft_complex<Tval>* output,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset)
|
|
{
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partitions.size())
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
output[odx + ooffset[0]]
|
|
= rocfft_complex<Tval>(input0[idx + ioffset[0]], input1[idx + ioffset[1]]);
|
|
} while(increment_rowmajor(index, length));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Copy data of dimensions length with strides istride and length idist between batches to
|
|
// a buffer with strides ostride and length odist between batches. The input type is
|
|
// complex interleaved and the output type is planar.
|
|
template <typename Tval, typename Tint1, typename Tint2, typename Tint3>
|
|
inline void copy_buffers_1to2(const rocfft_complex<Tval>* input,
|
|
Tval* output0,
|
|
Tval* output1,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset)
|
|
{
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for num_threads(partitions.size())
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
output0[odx + ooffset[0]] = input[idx + ioffset[0]].real();
|
|
output1[odx + ooffset[1]] = input[idx + ioffset[0]].imag();
|
|
} while(increment_rowmajor(index, length));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Copy data of dimensions length with strides istride and length idist between batches to
|
|
// a buffer with strides ostride and length odist between batches. The input type given
|
|
// by itype, and the output type is given by otype.
|
|
template <typename Tint1, typename Tint2, typename Tint3>
|
|
inline void copy_buffers(const std::vector<hostbuf>& input,
|
|
std::vector<hostbuf>& output,
|
|
const Tint1& length,
|
|
const size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type itype,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const fft_array_type otype,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset)
|
|
{
|
|
if(itype == otype)
|
|
{
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
copy_buffers_1to1(
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(input[0].data()),
|
|
reinterpret_cast<rocfft_complex<rocfft_fp16>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_single:
|
|
copy_buffers_1to1(reinterpret_cast<const rocfft_complex<float>*>(input[0].data()),
|
|
reinterpret_cast<rocfft_complex<float>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_double:
|
|
copy_buffers_1to1(reinterpret_cast<const rocfft_complex<double>*>(input[0].data()),
|
|
reinterpret_cast<rocfft_complex<double>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
}
|
|
break;
|
|
case fft_array_type_real:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
for(unsigned int idx = 0; idx < input.size(); ++idx)
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
copy_buffers_1to1(reinterpret_cast<const rocfft_fp16*>(input[idx].data()),
|
|
reinterpret_cast<rocfft_fp16*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_single:
|
|
copy_buffers_1to1(reinterpret_cast<const float*>(input[idx].data()),
|
|
reinterpret_cast<float*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_double:
|
|
copy_buffers_1to1(reinterpret_cast<const double*>(input[idx].data()),
|
|
reinterpret_cast<double*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid data type");
|
|
}
|
|
}
|
|
else if((itype == fft_array_type_complex_interleaved && otype == fft_array_type_complex_planar)
|
|
|| (itype == fft_array_type_hermitian_interleaved
|
|
&& otype == fft_array_type_hermitian_planar))
|
|
{
|
|
// copy 1to2
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
copy_buffers_1to2(reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(input[0].data()),
|
|
reinterpret_cast<rocfft_fp16*>(output[0].data()),
|
|
reinterpret_cast<rocfft_fp16*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_single:
|
|
copy_buffers_1to2(reinterpret_cast<const rocfft_complex<float>*>(input[0].data()),
|
|
reinterpret_cast<float*>(output[0].data()),
|
|
reinterpret_cast<float*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_double:
|
|
copy_buffers_1to2(reinterpret_cast<const rocfft_complex<double>*>(input[0].data()),
|
|
reinterpret_cast<double*>(output[0].data()),
|
|
reinterpret_cast<double*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
}
|
|
}
|
|
else if((itype == fft_array_type_complex_planar && otype == fft_array_type_complex_interleaved)
|
|
|| (itype == fft_array_type_hermitian_planar
|
|
&& otype == fft_array_type_hermitian_interleaved))
|
|
{
|
|
// copy 2 to 1
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
copy_buffers_2to1(reinterpret_cast<const rocfft_fp16*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(input[1].data()),
|
|
reinterpret_cast<rocfft_complex<rocfft_fp16>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_single:
|
|
copy_buffers_2to1(reinterpret_cast<const float*>(input[0].data()),
|
|
reinterpret_cast<const float*>(input[1].data()),
|
|
reinterpret_cast<rocfft_complex<float>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
case fft_precision_double:
|
|
copy_buffers_2to1(reinterpret_cast<const double*>(input[0].data()),
|
|
reinterpret_cast<const double*>(input[1].data()),
|
|
reinterpret_cast<rocfft_complex<double>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw std::runtime_error("Invalid input and output types.");
|
|
}
|
|
}
|
|
|
|
// unroll arbitrary-dimension copy_buffers into specializations for 1-, 2-, 3-dimensions
|
|
template <typename Tint1, typename Tint2, typename Tint3>
|
|
inline void copy_buffers(const std::vector<hostbuf>& input,
|
|
std::vector<hostbuf>& output,
|
|
const std::vector<Tint1>& length,
|
|
const size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type itype,
|
|
const std::vector<Tint2>& istride,
|
|
const size_t idist,
|
|
const fft_array_type otype,
|
|
const std::vector<Tint3>& ostride,
|
|
const size_t odist,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset)
|
|
{
|
|
switch(length.size())
|
|
{
|
|
case 1:
|
|
return copy_buffers(input,
|
|
output,
|
|
length[0],
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
istride[0],
|
|
idist,
|
|
otype,
|
|
ostride[0],
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
case 2:
|
|
return copy_buffers(input,
|
|
output,
|
|
std::make_tuple(length[0], length[1]),
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
std::make_tuple(istride[0], istride[1]),
|
|
idist,
|
|
otype,
|
|
std::make_tuple(ostride[0], ostride[1]),
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
case 3:
|
|
return copy_buffers(input,
|
|
output,
|
|
std::make_tuple(length[0], length[1], length[2]),
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
std::make_tuple(istride[0], istride[1], istride[2]),
|
|
idist,
|
|
otype,
|
|
std::make_tuple(ostride[0], ostride[1], ostride[2]),
|
|
odist,
|
|
ioffset,
|
|
ooffset);
|
|
case 4:
|
|
{
|
|
// treat 4D brick as 3D + batch, but this needs to be
|
|
// unbatched until we add 4D support for copy_buffers
|
|
if(nbatch != 1)
|
|
throw std::runtime_error("cannot copy batched 4D bricks");
|
|
return copy_buffers(input,
|
|
output,
|
|
std::make_tuple(length[1], length[2], length[3]),
|
|
length[0],
|
|
precision,
|
|
itype,
|
|
std::make_tuple(istride[1], istride[2], istride[3]),
|
|
istride[0],
|
|
otype,
|
|
std::make_tuple(ostride[1], ostride[2], istride[3]),
|
|
ostride[0],
|
|
ioffset,
|
|
ooffset);
|
|
}
|
|
default:
|
|
abort();
|
|
}
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 distance between two buffers with strides istride and
|
|
// length idist between batches to a buffer with strides ostride and length odist between
|
|
// batches. Both buffers are of complex type.
|
|
|
|
struct VectorNorms
|
|
{
|
|
double l_2 = 0.0, l_inf = 0.0;
|
|
};
|
|
|
|
template <typename Tcomplex, typename Tint1, typename Tint2, typename Tint3>
|
|
inline VectorNorms distance_1to1_complex(const Tcomplex* input,
|
|
const Tcomplex* output,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
std::vector<std::pair<size_t, size_t>>* linf_failures,
|
|
const double linf_cutoff,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset,
|
|
const double output_scalar = 1.0)
|
|
{
|
|
double linf = 0.0;
|
|
double l2 = 0.0;
|
|
|
|
std::mutex linf_failure_lock;
|
|
std::vector<std::pair<size_t, size_t>> linf_failures_private;
|
|
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_colmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for reduction(max : linf) reduction(+ : l2) \
|
|
num_threads(partitions.size()) private(linf_failures_private)
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
double cur_linf = 0.0;
|
|
double cur_l2 = 0.0;
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
const double rdiff
|
|
= std::abs(static_cast<double>(output[odx + ooffset[0]].real()) * output_scalar
|
|
- static_cast<double>(input[idx + ioffset[0]].real()));
|
|
if(rdiff > linf_cutoff)
|
|
{
|
|
std::pair<size_t, size_t> fval(b, idx);
|
|
if(linf_failures)
|
|
linf_failures_private.push_back(fval);
|
|
}
|
|
cur_linf = std::max(rdiff, cur_linf);
|
|
cur_l2 += rdiff * rdiff;
|
|
|
|
const double idiff
|
|
= std::abs(static_cast<double>(output[odx + ooffset[0]].imag()) * output_scalar
|
|
- static_cast<double>(input[idx + ioffset[0]].imag()));
|
|
if(idiff > linf_cutoff)
|
|
{
|
|
std::pair<size_t, size_t> fval(b, idx);
|
|
if(linf_failures)
|
|
linf_failures_private.push_back(fval);
|
|
}
|
|
cur_linf = std::max(idiff, cur_linf);
|
|
cur_l2 += idiff * idiff;
|
|
|
|
} while(increment_rowmajor(index, length));
|
|
linf = std::max(linf, cur_linf);
|
|
l2 += cur_l2;
|
|
|
|
if(linf_failures)
|
|
{
|
|
linf_failure_lock.lock();
|
|
std::copy(linf_failures_private.begin(),
|
|
linf_failures_private.end(),
|
|
std::back_inserter(*linf_failures));
|
|
linf_failure_lock.unlock();
|
|
}
|
|
}
|
|
}
|
|
return {.l_2 = sqrt(l2), .l_inf = linf};
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 distance between two buffers with strides istride and
|
|
// length idist between batches to a buffer with strides ostride and length odist between
|
|
// batches. Both buffers are of real type.
|
|
template <typename Tfloat, typename Tint1, typename Tint2, typename Tint3>
|
|
inline VectorNorms distance_1to1_real(const Tfloat* input,
|
|
const Tfloat* output,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
std::vector<std::pair<size_t, size_t>>* linf_failures,
|
|
const double linf_cutoff,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset,
|
|
const double output_scalar = 1.0)
|
|
{
|
|
double linf = 0.0;
|
|
double l2 = 0.0;
|
|
|
|
std::mutex linf_failure_lock;
|
|
std::vector<std::pair<size_t, size_t>> linf_failures_private;
|
|
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for reduction(max : linf) reduction(+ : l2) \
|
|
num_threads(partitions.size()) private(linf_failures_private)
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
double cur_linf = 0.0;
|
|
double cur_l2 = 0.0;
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
const double diff
|
|
= std::abs(static_cast<double>(output[odx + ooffset[0]]) * output_scalar
|
|
- static_cast<double>(input[idx + ioffset[0]]));
|
|
if(diff > linf_cutoff)
|
|
{
|
|
std::pair<size_t, size_t> fval(b, idx);
|
|
if(linf_failures)
|
|
linf_failures_private.push_back(fval);
|
|
}
|
|
cur_linf = std::max(diff, cur_linf);
|
|
cur_l2 += diff * diff;
|
|
|
|
} while(increment_rowmajor(index, length));
|
|
linf = std::max(linf, cur_linf);
|
|
l2 += cur_l2;
|
|
|
|
if(linf_failures)
|
|
{
|
|
linf_failure_lock.lock();
|
|
std::copy(linf_failures_private.begin(),
|
|
linf_failures_private.end(),
|
|
std::back_inserter(*linf_failures));
|
|
linf_failure_lock.unlock();
|
|
}
|
|
}
|
|
}
|
|
return {.l_2 = sqrt(l2), .l_inf = linf};
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 distance between two buffers with strides istride and
|
|
// length idist between batches to a buffer with strides ostride and length odist between
|
|
// batches. input is complex-interleaved, output is complex-planar.
|
|
template <typename Tval, typename Tint1, typename T2, typename T3>
|
|
inline VectorNorms distance_1to2(const rocfft_complex<Tval>* input,
|
|
const Tval* output0,
|
|
const Tval* output1,
|
|
const Tint1& whole_length,
|
|
const size_t nbatch,
|
|
const T2& istride,
|
|
const size_t idist,
|
|
const T3& ostride,
|
|
const size_t odist,
|
|
std::vector<std::pair<size_t, size_t>>* linf_failures,
|
|
const double linf_cutoff,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset,
|
|
const double output_scalar = 1.0)
|
|
{
|
|
double linf = 0.0;
|
|
double l2 = 0.0;
|
|
|
|
std::mutex linf_failure_lock;
|
|
std::vector<std::pair<size_t, size_t>> linf_failures_private;
|
|
|
|
const bool idx_equals_odx = istride == ostride && idist == odist;
|
|
size_t idx_base = 0;
|
|
size_t odx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist, odx_base += odist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for reduction(max : linf) reduction(+ : l2) \
|
|
num_threads(partitions.size()) private(linf_failures_private)
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
double cur_linf = 0.0;
|
|
double cur_l2 = 0.0;
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const auto odx = idx_equals_odx ? idx : compute_index(index, ostride, odx_base);
|
|
const double rdiff
|
|
= std::abs(static_cast<double>(output0[odx + ooffset[0]]) * output_scalar
|
|
- static_cast<double>(input[idx + ioffset[0]].real()));
|
|
if(rdiff > linf_cutoff)
|
|
{
|
|
std::pair<size_t, size_t> fval(b, idx);
|
|
if(linf_failures)
|
|
linf_failures_private.push_back(fval);
|
|
}
|
|
cur_linf = std::max(rdiff, cur_linf);
|
|
cur_l2 += rdiff * rdiff;
|
|
|
|
const double idiff
|
|
= std::abs(static_cast<double>(output1[odx + ooffset[1]]) * output_scalar
|
|
- static_cast<double>(input[idx + ioffset[0]].imag()));
|
|
if(idiff > linf_cutoff)
|
|
{
|
|
std::pair<size_t, size_t> fval(b, idx);
|
|
if(linf_failures)
|
|
linf_failures_private.push_back(fval);
|
|
}
|
|
cur_linf = std::max(idiff, cur_linf);
|
|
cur_l2 += idiff * idiff;
|
|
|
|
} while(increment_rowmajor(index, length));
|
|
linf = std::max(linf, cur_linf);
|
|
l2 += cur_l2;
|
|
|
|
if(linf_failures)
|
|
{
|
|
linf_failure_lock.lock();
|
|
std::copy(linf_failures_private.begin(),
|
|
linf_failures_private.end(),
|
|
std::back_inserter(*linf_failures));
|
|
linf_failure_lock.unlock();
|
|
}
|
|
}
|
|
}
|
|
return {.l_2 = sqrt(l2), .l_inf = linf};
|
|
}
|
|
|
|
// Compute the L-inifnity and L-2 distance between two buffers of dimension length and
|
|
// with types given by itype, otype, and precision.
|
|
template <typename Tint1, typename Tint2, typename Tint3>
|
|
inline VectorNorms distance(const std::vector<hostbuf>& input,
|
|
const std::vector<hostbuf>& output,
|
|
const Tint1& length,
|
|
const size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type itype,
|
|
const Tint2& istride,
|
|
const size_t idist,
|
|
const fft_array_type otype,
|
|
const Tint3& ostride,
|
|
const size_t odist,
|
|
std::vector<std::pair<size_t, size_t>>* linf_failures,
|
|
const double linf_cutoff,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset,
|
|
const double output_scalar = 1.0)
|
|
{
|
|
VectorNorms dist;
|
|
|
|
if(itype == otype)
|
|
{
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
dist = distance_1to1_complex(
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_single:
|
|
dist = distance_1to1_complex(
|
|
reinterpret_cast<const rocfft_complex<float>*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_complex<float>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_double:
|
|
dist = distance_1to1_complex(
|
|
reinterpret_cast<const rocfft_complex<double>*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_complex<double>*>(output[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
}
|
|
dist.l_2 *= dist.l_2;
|
|
break;
|
|
case fft_array_type_real:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
for(unsigned int idx = 0; idx < input.size(); ++idx)
|
|
{
|
|
VectorNorms d;
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
d = distance_1to1_real(reinterpret_cast<const rocfft_fp16*>(input[idx].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_single:
|
|
d = distance_1to1_real(reinterpret_cast<const float*>(input[idx].data()),
|
|
reinterpret_cast<const float*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_double:
|
|
d = distance_1to1_real(reinterpret_cast<const double*>(input[idx].data()),
|
|
reinterpret_cast<const double*>(output[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
}
|
|
dist.l_inf = std::max(d.l_inf, dist.l_inf);
|
|
dist.l_2 += d.l_2 * d.l_2;
|
|
}
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid input and output types.");
|
|
}
|
|
}
|
|
else if((itype == fft_array_type_complex_interleaved && otype == fft_array_type_complex_planar)
|
|
|| (itype == fft_array_type_hermitian_interleaved
|
|
&& otype == fft_array_type_hermitian_planar))
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
dist = distance_1to2(
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(output[0].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_single:
|
|
dist = distance_1to2(reinterpret_cast<const rocfft_complex<float>*>(input[0].data()),
|
|
reinterpret_cast<const float*>(output[0].data()),
|
|
reinterpret_cast<const float*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_double:
|
|
dist = distance_1to2(reinterpret_cast<const rocfft_complex<double>*>(input[0].data()),
|
|
reinterpret_cast<const double*>(output[0].data()),
|
|
reinterpret_cast<const double*>(output[1].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
ostride,
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
}
|
|
dist.l_2 *= dist.l_2;
|
|
}
|
|
else if((itype == fft_array_type_complex_planar && otype == fft_array_type_complex_interleaved)
|
|
|| (itype == fft_array_type_hermitian_planar
|
|
&& otype == fft_array_type_hermitian_interleaved))
|
|
{
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
dist = distance_1to2(
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(output[0].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(input[0].data()),
|
|
reinterpret_cast<const rocfft_fp16*>(input[1].data()),
|
|
length,
|
|
nbatch,
|
|
ostride,
|
|
odist,
|
|
istride,
|
|
idist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_single:
|
|
dist = distance_1to2(reinterpret_cast<const rocfft_complex<float>*>(output[0].data()),
|
|
reinterpret_cast<const float*>(input[0].data()),
|
|
reinterpret_cast<const float*>(input[1].data()),
|
|
length,
|
|
nbatch,
|
|
ostride,
|
|
odist,
|
|
istride,
|
|
idist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
case fft_precision_double:
|
|
dist = distance_1to2(reinterpret_cast<const rocfft_complex<double>*>(output[0].data()),
|
|
reinterpret_cast<const double*>(input[0].data()),
|
|
reinterpret_cast<const double*>(input[1].data()),
|
|
length,
|
|
nbatch,
|
|
ostride,
|
|
odist,
|
|
istride,
|
|
idist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
break;
|
|
}
|
|
dist.l_2 *= dist.l_2;
|
|
}
|
|
else
|
|
{
|
|
throw std::runtime_error("Invalid input and output types.");
|
|
}
|
|
dist.l_2 = sqrt(dist.l_2);
|
|
return dist;
|
|
}
|
|
|
|
// check if the specified length + stride/dist is contiguous
|
|
template <typename Tint1, typename Tint2>
|
|
bool is_contiguous_rowmajor(const std::vector<Tint1>& length,
|
|
const std::vector<Tint2>& stride,
|
|
size_t dist)
|
|
{
|
|
size_t expected_stride = 1;
|
|
auto stride_it = stride.rbegin();
|
|
auto length_it = length.rbegin();
|
|
for(; stride_it != stride.rend() && length_it != length.rend(); ++stride_it, ++length_it)
|
|
{
|
|
if(*stride_it != expected_stride)
|
|
return false;
|
|
expected_stride *= *length_it;
|
|
}
|
|
return expected_stride == dist;
|
|
}
|
|
|
|
// Unroll arbitrary-dimension distance into specializations for 1-, 2-, 3-dimensions
|
|
template <typename Tint1, typename Tint2, typename Tint3>
|
|
inline VectorNorms distance(const std::vector<hostbuf>& input,
|
|
const std::vector<hostbuf>& output,
|
|
std::vector<Tint1> length,
|
|
size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type itype,
|
|
std::vector<Tint2> istride,
|
|
const size_t idist,
|
|
const fft_array_type otype,
|
|
std::vector<Tint3> ostride,
|
|
const size_t odist,
|
|
std::vector<std::pair<size_t, size_t>>* linf_failures,
|
|
const double linf_cutoff,
|
|
const std::vector<size_t>& ioffset,
|
|
const std::vector<size_t>& ooffset,
|
|
const double output_scalar = 1.0)
|
|
{
|
|
// If istride and ostride are both contiguous, collapse them down
|
|
// to one dimension. Index calculation is simpler (and faster)
|
|
// in the 1D case.
|
|
if(is_contiguous_rowmajor(length, istride, idist)
|
|
&& is_contiguous_rowmajor(length, ostride, odist))
|
|
{
|
|
length = {product(length.begin(), length.end()) * nbatch};
|
|
istride = {static_cast<Tint2>(1)};
|
|
ostride = {static_cast<Tint3>(1)};
|
|
nbatch = 1;
|
|
}
|
|
|
|
switch(length.size())
|
|
{
|
|
case 1:
|
|
return distance(input,
|
|
output,
|
|
length[0],
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
istride[0],
|
|
idist,
|
|
otype,
|
|
ostride[0],
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
case 2:
|
|
return distance(input,
|
|
output,
|
|
std::make_tuple(length[0], length[1]),
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
std::make_tuple(istride[0], istride[1]),
|
|
idist,
|
|
otype,
|
|
std::make_tuple(ostride[0], ostride[1]),
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
case 3:
|
|
return distance(input,
|
|
output,
|
|
std::make_tuple(length[0], length[1], length[2]),
|
|
nbatch,
|
|
precision,
|
|
itype,
|
|
std::make_tuple(istride[0], istride[1], istride[2]),
|
|
idist,
|
|
otype,
|
|
std::make_tuple(ostride[0], ostride[1], ostride[2]),
|
|
odist,
|
|
linf_failures,
|
|
linf_cutoff,
|
|
ioffset,
|
|
ooffset,
|
|
output_scalar);
|
|
default:
|
|
abort();
|
|
}
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 norm of a buffer with strides istride and
|
|
// length idist. Data is rocfft_complex.
|
|
template <typename Tcomplex, typename T1, typename T2>
|
|
inline VectorNorms norm_complex(const Tcomplex* input,
|
|
const T1& whole_length,
|
|
const size_t nbatch,
|
|
const T2& istride,
|
|
const size_t idist,
|
|
const std::vector<size_t>& offset)
|
|
{
|
|
double linf = 0.0;
|
|
double l2 = 0.0;
|
|
|
|
size_t idx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for reduction(max : linf) reduction(+ : l2) num_threads(partitions.size())
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
double cur_linf = 0.0;
|
|
double cur_l2 = 0.0;
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
|
|
const double rval = std::abs(static_cast<double>(input[idx + offset[0]].real()));
|
|
cur_linf = std::max(rval, cur_linf);
|
|
cur_l2 += rval * rval;
|
|
|
|
const double ival = std::abs(static_cast<double>(input[idx + offset[0]].imag()));
|
|
cur_linf = std::max(ival, cur_linf);
|
|
cur_l2 += ival * ival;
|
|
|
|
} while(increment_rowmajor(index, length));
|
|
linf = std::max(linf, cur_linf);
|
|
l2 += cur_l2;
|
|
}
|
|
}
|
|
return {.l_2 = sqrt(l2), .l_inf = linf};
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 norm of abuffer with strides istride and
|
|
// length idist. Data is real-valued.
|
|
template <typename Tfloat, typename T1, typename T2>
|
|
inline VectorNorms norm_real(const Tfloat* input,
|
|
const T1& whole_length,
|
|
const size_t nbatch,
|
|
const T2& istride,
|
|
const size_t idist,
|
|
const std::vector<size_t>& offset)
|
|
{
|
|
double linf = 0.0;
|
|
double l2 = 0.0;
|
|
|
|
size_t idx_base = 0;
|
|
auto partitions = partition_rowmajor(whole_length);
|
|
for(size_t b = 0; b < nbatch; b++, idx_base += idist)
|
|
{
|
|
#ifdef _OPENMP
|
|
#pragma omp parallel for reduction(max : linf) reduction(+ : l2) num_threads(partitions.size())
|
|
#endif
|
|
for(size_t part = 0; part < partitions.size(); ++part)
|
|
{
|
|
double cur_linf = 0.0;
|
|
double cur_l2 = 0.0;
|
|
auto index = partitions[part].first;
|
|
const auto length = partitions[part].second;
|
|
do
|
|
{
|
|
const auto idx = compute_index(index, istride, idx_base);
|
|
const double val = std::abs(static_cast<double>(input[idx + offset[0]]));
|
|
cur_linf = std::max(val, cur_linf);
|
|
cur_l2 += val * val;
|
|
|
|
} while(increment_rowmajor(index, length));
|
|
linf = std::max(linf, cur_linf);
|
|
l2 += cur_l2;
|
|
}
|
|
}
|
|
return {.l_2 = sqrt(l2), .l_inf = linf};
|
|
}
|
|
|
|
// Compute the L-infinity and L-2 norm of abuffer with strides istride and
|
|
// length idist. Data format is given by precision and itype.
|
|
template <typename T1, typename T2>
|
|
inline VectorNorms norm(const std::vector<hostbuf>& input,
|
|
const T1& length,
|
|
const size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type itype,
|
|
const T2& istride,
|
|
const size_t idist,
|
|
const std::vector<size_t>& offset)
|
|
{
|
|
VectorNorms norm;
|
|
|
|
switch(itype)
|
|
{
|
|
case fft_array_type_complex_interleaved:
|
|
case fft_array_type_hermitian_interleaved:
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
norm = norm_complex(
|
|
reinterpret_cast<const rocfft_complex<rocfft_fp16>*>(input[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
case fft_precision_single:
|
|
norm = norm_complex(reinterpret_cast<const rocfft_complex<float>*>(input[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
case fft_precision_double:
|
|
norm = norm_complex(reinterpret_cast<const rocfft_complex<double>*>(input[0].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
}
|
|
norm.l_2 *= norm.l_2;
|
|
break;
|
|
case fft_array_type_real:
|
|
case fft_array_type_complex_planar:
|
|
case fft_array_type_hermitian_planar:
|
|
for(unsigned int idx = 0; idx < input.size(); ++idx)
|
|
{
|
|
VectorNorms n;
|
|
switch(precision)
|
|
{
|
|
case fft_precision_half:
|
|
n = norm_real(reinterpret_cast<const rocfft_fp16*>(input[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
case fft_precision_single:
|
|
n = norm_real(reinterpret_cast<const float*>(input[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
case fft_precision_double:
|
|
n = norm_real(reinterpret_cast<const double*>(input[idx].data()),
|
|
length,
|
|
nbatch,
|
|
istride,
|
|
idist,
|
|
offset);
|
|
break;
|
|
}
|
|
norm.l_inf = std::max(n.l_inf, norm.l_inf);
|
|
norm.l_2 += n.l_2 * n.l_2;
|
|
}
|
|
break;
|
|
default:
|
|
throw std::runtime_error("Invalid data type");
|
|
}
|
|
|
|
norm.l_2 = sqrt(norm.l_2);
|
|
return norm;
|
|
}
|
|
|
|
// Unroll arbitrary-dimension norm into specializations for 1-, 2-, 3-dimensions
|
|
template <typename T1, typename T2>
|
|
inline VectorNorms norm(const std::vector<hostbuf>& input,
|
|
std::vector<T1> length,
|
|
size_t nbatch,
|
|
const fft_precision precision,
|
|
const fft_array_type type,
|
|
std::vector<T2> stride,
|
|
const size_t dist,
|
|
const std::vector<size_t>& offset)
|
|
{
|
|
// If stride is contiguous, collapse it down to one dimension.
|
|
// Index calculation is simpler (and faster) in the 1D case.
|
|
if(is_contiguous_rowmajor(length, stride, dist))
|
|
{
|
|
length = {product(length.begin(), length.end()) * nbatch};
|
|
stride = {static_cast<T2>(1)};
|
|
nbatch = 1;
|
|
}
|
|
|
|
switch(length.size())
|
|
{
|
|
case 1:
|
|
return norm(input, length[0], nbatch, precision, type, stride[0], dist, offset);
|
|
case 2:
|
|
return norm(input,
|
|
std::make_tuple(length[0], length[1]),
|
|
nbatch,
|
|
precision,
|
|
type,
|
|
std::make_tuple(stride[0], stride[1]),
|
|
dist,
|
|
offset);
|
|
case 3:
|
|
return norm(input,
|
|
std::make_tuple(length[0], length[1], length[2]),
|
|
nbatch,
|
|
precision,
|
|
type,
|
|
std::make_tuple(stride[0], stride[1], stride[2]),
|
|
dist,
|
|
offset);
|
|
default:
|
|
abort();
|
|
}
|
|
}
|
|
|
|
// returns byte_sizes.size() host buffers of respective sizes byte_sizes[0], byte_sizes[1], ...
|
|
static std::vector<hostbuf> allocate_host_buffer(const std::vector<size_t>& byte_sizes)
|
|
{
|
|
std::vector<hostbuf> buffers(byte_sizes.size());
|
|
for(unsigned int i = 0; i < byte_sizes.size(); ++i)
|
|
{
|
|
buffers[i].alloc(byte_sizes[i]);
|
|
}
|
|
return buffers;
|
|
}
|
|
|
|
static std::vector<hostbuf> allocate_host_buffer(const fft_precision precision,
|
|
const fft_array_type type,
|
|
const std::vector<size_t>& size)
|
|
{
|
|
std::vector<size_t> byte_sizes = size;
|
|
for(auto& sz : byte_sizes)
|
|
sz *= var_size<size_t>(precision, type);
|
|
return allocate_host_buffer(byte_sizes);
|
|
}
|
|
|
|
// Check if the required buffers fit in the device vram.
|
|
inline bool vram_fits_problem(const std::vector<size_t>& prob_size,
|
|
const std::vector<size_t>& vram_avail)
|
|
{
|
|
if(prob_size.size() != vram_avail.size())
|
|
throw std::runtime_error("prob/vram size mismatch");
|
|
|
|
// We keep a small margin of error for fitting the problem into vram:
|
|
const size_t extra = 1 << 27;
|
|
|
|
for(size_t i = 0; i < prob_size.size(); ++i)
|
|
{
|
|
if(prob_size[i] + extra > vram_avail[i])
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// set input for a brick in a field
|
|
// functor to search for bricks on a rank, in a container of bricks
|
|
// sorted by rank
|
|
struct match_rank
|
|
{
|
|
bool operator()(const fft_params::fft_brick& b, int rank) const
|
|
{
|
|
return b.rank < rank;
|
|
}
|
|
bool operator()(int rank, const fft_params::fft_brick& b) const
|
|
{
|
|
return rank < b.rank;
|
|
}
|
|
};
|
|
|
|
// Initialize input for the bricks on the specified comm rank
|
|
// (assumed to be the local rank)
|
|
template <typename Tparams, typename Tbuff>
|
|
void init_local_input(int comm_rank,
|
|
const Tparams& params,
|
|
const std::vector<fft_params::fft_brick>& bricks,
|
|
size_t elem_size,
|
|
const std::vector<void*>& input_ptrs)
|
|
{
|
|
// get bricks for this rank
|
|
auto range = std::equal_range(bricks.begin(), bricks.end(), comm_rank, match_rank());
|
|
|
|
const bool is_planar = params.itype == fft_array_type_complex_planar
|
|
|| params.itype == fft_array_type_hermitian_planar;
|
|
|
|
size_t ptr_idx = 0;
|
|
for(auto brick = range.first; brick != range.second; ++brick, ++ptr_idx)
|
|
{
|
|
rocfft_scoped_device dev(brick->device);
|
|
|
|
// some utility code below needs batch separated from brick lengths
|
|
std::vector<size_t> brick_len_nobatch = brick->length();
|
|
auto brick_batch = brick_len_nobatch.front();
|
|
brick_len_nobatch.erase(brick_len_nobatch.begin());
|
|
std::vector<size_t> brick_stride_nobatch = brick->stride;
|
|
auto brick_dist = brick_stride_nobatch.front();
|
|
brick_stride_nobatch.erase(brick_stride_nobatch.begin());
|
|
std::vector<size_t> brick_lower_nobatch = brick->lower;
|
|
auto brick_lower_batch = brick_lower_nobatch.front();
|
|
brick_lower_nobatch.erase(brick_lower_nobatch.begin());
|
|
|
|
auto contiguous_stride = params.compute_stride(params.ilength());
|
|
auto contiguous_dist = params.compute_idist();
|
|
|
|
std::vector<Tbuff> bufvec(1);
|
|
size_t brick_size_bytes
|
|
= compute_ptrdiff(brick->length(), brick->stride) * elem_size / (is_planar ? 2 : 1);
|
|
bufvec.back() = Tbuff::make_nonowned(input_ptrs[ptr_idx], brick_size_bytes);
|
|
// grab a second pointer for planar
|
|
if(is_planar)
|
|
{
|
|
++ptr_idx;
|
|
bufvec.push_back(Tbuff::make_nonowned(input_ptrs[ptr_idx], brick_size_bytes));
|
|
}
|
|
|
|
// generate data (in device mem)
|
|
switch(params.precision)
|
|
{
|
|
case fft_precision_half:
|
|
set_input<Tbuff, rocfft_fp16>(bufvec,
|
|
params.igen,
|
|
params.itype,
|
|
brick_len_nobatch,
|
|
brick_len_nobatch,
|
|
brick_stride_nobatch,
|
|
params.ioffset,
|
|
brick_dist,
|
|
brick_batch,
|
|
get_curr_device_prop(),
|
|
brick_lower_nobatch,
|
|
brick_lower_batch,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
case fft_precision_single:
|
|
set_input<Tbuff, float>(bufvec,
|
|
params.igen,
|
|
params.itype,
|
|
brick_len_nobatch,
|
|
brick_len_nobatch,
|
|
brick_stride_nobatch,
|
|
params.ioffset,
|
|
brick_dist,
|
|
brick_batch,
|
|
get_curr_device_prop(),
|
|
brick_lower_nobatch,
|
|
brick_lower_batch,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
case fft_precision_double:
|
|
set_input<Tbuff, double>(bufvec,
|
|
params.igen,
|
|
params.itype,
|
|
brick_len_nobatch,
|
|
brick_len_nobatch,
|
|
brick_stride_nobatch,
|
|
params.ioffset,
|
|
brick_dist,
|
|
brick_batch,
|
|
get_curr_device_prop(),
|
|
brick_lower_nobatch,
|
|
brick_lower_batch,
|
|
contiguous_stride,
|
|
contiguous_dist);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
#endif
|