derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
687 lines
40 KiB
C++
687 lines
40 KiB
C++
// Copyright (C) 2021 - 2026 Advanced Micro Devices, Inc. All rights reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
// THE SOFTWARE.
|
|
|
|
#include "../../shared/accuracy_test.h"
|
|
#include "../../shared/params_gen.h"
|
|
|
|
std::vector<std::vector<size_t>> adhoc_sizes = {
|
|
// sizes that exercise L1D_TRTRT subplan of 2D_RTRT or 3D_TRTRTR
|
|
{1, 220},
|
|
{1, 330},
|
|
{81, 220, 36},
|
|
|
|
// L1D_CC subplan of 3D_TRTRTR
|
|
{4, 4, 8192},
|
|
|
|
// SBRC 192 with special param
|
|
{192, 192, 192},
|
|
{192, 84, 84},
|
|
|
|
// Failure with build_CS_3D_BLOCK_RC
|
|
{680, 128, 128},
|
|
|
|
// Large 1D primes that fall above the block threshold (length 262144).
|
|
// Bluestein requires two forwards and one inverse FFTs, and the plan
|
|
// for these sizes breakdown these FFTs either as:
|
|
// L1D_TRTRT (T + STOCKHAM + T + STOCKHAM + T) for lengthBlue <= 4096^2
|
|
// or
|
|
// L1D_TRTRT (T + L1D_CC + STOCKHAM_BL_CC + STOCHMAM_BL_RC + T + STOCKHAM + T)
|
|
// for lengthBlue > 4096^2.
|
|
{196597},
|
|
{25165813},
|
|
|
|
// 2D single-kernel bluestein size combined with multi-kernel bluestein
|
|
{19, 2053},
|
|
|
|
// TILE_UNALIGNED type of SBRC 3D ERC
|
|
{98, 98, 98},
|
|
|
|
// 3D_BLOCK_CR
|
|
{336, 336, 56},
|
|
|
|
// real-Stockham fusion boundary cases.
|
|
// 768, 1536, 3072 newly fuse on more LDS at sp/dp.
|
|
// 4096, 8192 verify the LDS-fit helper correctly refuses too-large fused
|
|
// kernels (negative test). param_generator covers complex_forward and
|
|
// real_forward for each entry, and the round-trip in accuracy_test.h
|
|
// exercises C2R automatically.
|
|
{768},
|
|
{1536},
|
|
{3072},
|
|
{4096},
|
|
{8192},
|
|
};
|
|
|
|
const static std::vector<std::vector<size_t>> stride_range = {{1}};
|
|
|
|
static std::vector<std::vector<size_t>> ioffset_range_zero = {{0, 0}};
|
|
static std::vector<std::vector<size_t>> ooffset_range_zero = {{0, 0}};
|
|
|
|
static std::vector<std::vector<size_t>> ioffset_range = {{0, 0}, {1, 1}};
|
|
static std::vector<std::vector<size_t>> ooffset_range = {{0, 0}, {1, 1}};
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator(test_prob,
|
|
adhoc_sizes,
|
|
precision_range_sp_dp,
|
|
batch_range,
|
|
stride_range,
|
|
stride_range,
|
|
ioffset_range_zero,
|
|
ooffset_range_zero,
|
|
place_range,
|
|
true)),
|
|
accuracy_test::TestName);
|
|
|
|
INSTANTIATE_TEST_SUITE_P(DISABLED_offset_adhoc,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator(test_prob,
|
|
adhoc_sizes,
|
|
precision_range_full,
|
|
batch_range,
|
|
stride_range,
|
|
stride_range,
|
|
ioffset_range,
|
|
ooffset_range,
|
|
place_range,
|
|
true)),
|
|
accuracy_test::TestName);
|
|
|
|
// Test that dist is ignored for batch-1 transforms. Normally,
|
|
// in-place transforms require same dist, but for batch-1 dist isn't
|
|
// used for anything and differing dist should be allowed.
|
|
inline auto param_permissive_iodist()
|
|
{
|
|
std::vector<std::vector<size_t>> lengths = adhoc_sizes;
|
|
lengths.push_back({4});
|
|
|
|
std::vector<fft_params> params;
|
|
for(const auto precision : precision_range_sp_dp)
|
|
{
|
|
for(const auto trans_type : trans_type_range)
|
|
{
|
|
for(const auto& types : generate_types(trans_type, place_range, true))
|
|
{
|
|
if(std::get<1>(types) != fft_placement_inplace)
|
|
continue;
|
|
for(const auto& len : lengths)
|
|
{
|
|
fft_params param;
|
|
|
|
param.length = len;
|
|
param.precision = precision;
|
|
param.idist = 2;
|
|
param.odist = 3;
|
|
param.transform_type = std::get<0>(types);
|
|
param.placement = std::get<1>(types);
|
|
param.itype = std::get<2>(types);
|
|
param.otype = std::get<3>(types);
|
|
|
|
param.validate();
|
|
|
|
const double roll = hash_prob(random_seed, param.token());
|
|
const double run_prob
|
|
= test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0)
|
|
* (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0)
|
|
* (param.is_real() ? real_prob_factor : 1.0)
|
|
* (param.is_callback() ? callback_prob_factor : 1.0);
|
|
|
|
if(roll > run_prob)
|
|
{
|
|
if(verbose > 4)
|
|
{
|
|
std::cout << "Test skipped (probability " << run_prob << " > " << roll
|
|
<< ")\n";
|
|
}
|
|
continue;
|
|
}
|
|
if(param.valid(0))
|
|
{
|
|
params.push_back(param);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return params;
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_dist,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_permissive_iodist()),
|
|
accuracy_test::TestName);
|
|
|
|
inline auto param_adhoc_colmajor()
|
|
{
|
|
// generate basic FFTs of adhoc sizes
|
|
auto params = param_generator(test_prob,
|
|
adhoc_sizes,
|
|
{fft_precision_single},
|
|
{2},
|
|
stride_range,
|
|
stride_range,
|
|
ioffset_range_zero,
|
|
ooffset_range_zero,
|
|
{fft_placement_notinplace},
|
|
false);
|
|
|
|
// remove any params that are:
|
|
// - 1D (not enough dims to swap)
|
|
// - real-complex 2D (we only get to play with higher dims, so
|
|
// again not enough dims to swap)
|
|
params.erase(std::remove_if(params.begin(),
|
|
params.end(),
|
|
[](const fft_params& param) {
|
|
if(param.length.size() == 1)
|
|
return true;
|
|
if(param.length.size() == 2)
|
|
{
|
|
if(param.transform_type == fft_transform_type_real_forward
|
|
|| param.transform_type
|
|
== fft_transform_type_real_inverse)
|
|
return true;
|
|
}
|
|
return false;
|
|
}),
|
|
params.end());
|
|
|
|
// reverse length/stride order on remaining params to make them
|
|
// col-major
|
|
std::for_each(params.begin(), params.end(), [](fft_params& param) {
|
|
size_t start_dim = 0;
|
|
// for real-complex we can't touch the fastest dim
|
|
if(param.transform_type == fft_transform_type_real_forward
|
|
|| param.transform_type == fft_transform_type_real_inverse)
|
|
++start_dim;
|
|
std::reverse(param.length.rbegin() + start_dim, param.length.rend());
|
|
std::reverse(param.istride.rbegin() + start_dim, param.istride.rend());
|
|
std::reverse(param.ostride.rbegin() + start_dim, param.ostride.rend());
|
|
});
|
|
return params;
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_colmajor,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_adhoc_colmajor()),
|
|
accuracy_test::TestName);
|
|
|
|
inline auto param_adhoc_stride()
|
|
{
|
|
std::vector<fft_params> params;
|
|
|
|
for(const auto precision : precision_range_full)
|
|
{
|
|
for(const auto& types : generate_types(fft_transform_type_complex_forward,
|
|
{fft_placement_inplace, fft_placement_notinplace},
|
|
true))
|
|
{
|
|
// 2D with non-contiguous strides and dist
|
|
fft_params param;
|
|
param.length = {2, 35};
|
|
param.precision = precision;
|
|
param.idist = 200;
|
|
param.odist = 200;
|
|
param.transform_type = fft_transform_type_complex_forward;
|
|
param.nbatch = 2;
|
|
param.placement = std::get<1>(types);
|
|
param.itype = std::get<2>(types);
|
|
param.otype = std::get<3>(types);
|
|
param.istride = {90, 2};
|
|
param.ostride = {90, 2};
|
|
param.validate();
|
|
const double roll = hash_prob(random_seed, param.token());
|
|
const double run_prob
|
|
= test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0)
|
|
* (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0)
|
|
* (param.is_real() ? real_prob_factor : 1.0)
|
|
* (param.is_callback() ? callback_prob_factor : 1.0);
|
|
if(roll > run_prob)
|
|
{
|
|
if(verbose > 4)
|
|
{
|
|
std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n";
|
|
}
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
if(param.valid(0))
|
|
{
|
|
params.push_back(param);
|
|
}
|
|
}
|
|
}
|
|
|
|
// test C2R/R2C with non-contiguous higher strides and dist - we
|
|
// want unit stride for length0 so we do the even-length optimization
|
|
for(const auto& types :
|
|
generate_types(fft_transform_type_real_forward, {fft_placement_notinplace}, true))
|
|
{
|
|
|
|
fft_params param;
|
|
param.length = {4, 4, 4};
|
|
param.precision = precision;
|
|
param.idist = 0;
|
|
param.odist = 0;
|
|
param.transform_type = fft_transform_type_real_forward;
|
|
param.nbatch = 2;
|
|
param.placement = std::get<1>(types);
|
|
param.itype = std::get<2>(types);
|
|
param.otype = std::get<3>(types);
|
|
param.istride = {16, 4, 1};
|
|
param.ostride = {16, 4, 1};
|
|
|
|
param.validate();
|
|
|
|
{
|
|
const double roll = hash_prob(random_seed, param.token());
|
|
const double run_prob
|
|
= test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0)
|
|
* (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0)
|
|
* (param.is_real() ? real_prob_factor : 1.0)
|
|
* (param.is_callback() ? callback_prob_factor : 1.0);
|
|
|
|
if(roll > run_prob)
|
|
{
|
|
if(verbose > 4)
|
|
{
|
|
std::cout << "Test skipped (probability " << run_prob << " > " << roll
|
|
<< ")\n";
|
|
}
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
if(param.valid(0))
|
|
{
|
|
params.push_back(param);
|
|
}
|
|
}
|
|
}
|
|
|
|
param.length = {2, 2, 2};
|
|
param.precision = precision;
|
|
param.idist = 0;
|
|
param.odist = 0;
|
|
param.transform_type = fft_transform_type_real_forward;
|
|
param.nbatch = 2;
|
|
param.placement = std::get<1>(types);
|
|
param.itype = std::get<2>(types);
|
|
param.otype = std::get<3>(types);
|
|
param.istride = {20, 6, 1};
|
|
param.ostride = {20, 6, 1};
|
|
|
|
param.validate();
|
|
|
|
{
|
|
const double roll = hash_prob(random_seed, param.token());
|
|
const double run_prob
|
|
= test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0)
|
|
* (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0)
|
|
* (param.is_real() ? real_prob_factor : 1.0)
|
|
* (param.is_callback() ? callback_prob_factor : 1.0);
|
|
|
|
if(roll > run_prob)
|
|
{
|
|
if(verbose > 4)
|
|
{
|
|
std::cout << "Test skipped (probability " << run_prob << " > " << roll
|
|
<< ")\n";
|
|
}
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
if(param.valid(0))
|
|
{
|
|
params.push_back(param);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return params;
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_stride,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_adhoc_stride()),
|
|
accuracy_test::TestName);
|
|
|
|
const auto adhoc_tokens = {
|
|
// clang-format off
|
|
"complex_forward_len_4_4_4_single_op_batch_2_istride_16_4_1_CI_ostride_4_16_1_CI_idist_64_odist_64_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_512_64_single_ip_batch_3_istride_192_3_CI_ostride_192_3_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_1024_1024_1024_single_op_batch_1_istride_1048576_1024_1_R_ostride_525312_513_1_HI_idist_1073741824_odist_537919488_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_6144_single_ip_batch_34_istride_35_CI_ostride_35_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_8192_single_ip_batch_65537_istride_1_R_ostride_1_HI_idist_8194_odist_4097_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_520_single_op_batch_270400_istride_1_R_ostride_1_HI_idist_520_odist_261_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_630_single_op_batch_396900_istride_1_R_ostride_1_HI_idist_630_odist_316_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_660_single_op_batch_435600_istride_1_R_ostride_1_HI_idist_660_odist_331_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_700_single_op_batch_490000_istride_1_R_ostride_1_HI_idist_700_odist_351_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_728_single_op_batch_529984_istride_1_R_ostride_1_HI_idist_728_odist_365_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_968_single_op_batch_937024_istride_1_R_ostride_1_HI_idist_968_odist_485_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_1020_single_op_batch_1040400_istride_1_R_ostride_1_HI_idist_1020_odist_511_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_378_42_single_ip_batch_66000_istride_44_1_R_ostride_22_1_HI_idist_16632_odist_8316_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_527_25_single_ip_batch_67500_istride_26_1_R_ostride_13_1_HI_idist_13702_odist_6851_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_630_38_single_ip_batch_65540_istride_40_1_R_ostride_20_1_HI_idist_25200_odist_12600_ioffset_0_0_ooffset_0_0",
|
|
// degenerate length-1 test cases
|
|
"complex_forward_len_1_single_ip_batch_2_istride_1_CI_ostride_1_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_1_single_op_batch_2_istride_1_CI_ostride_1_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_2_single_ip_batch_1_istride_1_R_ostride_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_2_single_op_batch_1_istride_1_R_ostride_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_1_2_single_ip_batch_2_istride_1_1_R_ostride_1_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_1_2_single_op_batch_2_istride_1_1_R_ostride_1_1_HI_idist_2_odist_2_ioffset_0_0_ooffset_0_0",
|
|
// clang-format on
|
|
};
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_token,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator_token(test_prob, adhoc_tokens)),
|
|
accuracy_test::TestName);
|
|
|
|
const auto adhoc_hermitian_input_tokens = {
|
|
// clang-format off
|
|
"real_inverse_len_1_8_double_ip_batch_1_istride_5_1_HI_ostride_10_1_R_idist_5_odist_10_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_8_double_ip_batch_10_istride_5_1_HI_ostride_10_1_R_idist_5_odist_10_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_8_1_double_ip_batch_1_istride_1_1_HI_ostride_2_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_8_1_double_ip_batch_10_istride_1_1_HI_ostride_2_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_2_7_double_ip_batch_1_istride_4_1_HI_ostride_8_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_7_2_double_ip_batch_1_istride_2_1_HI_ostride_4_1_R_idist_14_odist_28_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_16_512_double_ip_batch_1_istride_257_1_HI_ostride_514_1_R_idist_4112_odist_8224_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_16_512_double_ip_batch_10_istride_257_1_HI_ostride_514_1_R_idist_4112_odist_8224_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_512_16_double_ip_batch_1_istride_9_1_HI_ostride_18_1_R_idist_4608_odist_9216_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_512_16_double_ip_batch_10_istride_9_1_HI_ostride_18_1_R_idist_4608_odist_9216_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_1_1_double_ip_batch_1_istride_1_1_1_HI_ostride_2_2_1_R_idist_1_odist_2_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_1_1_double_ip_batch_10_istride_1_1_1_HI_ostride_2_2_1_R_idist_1_odist_2_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_2_1_double_ip_batch_1_istride_2_1_1_HI_ostride_4_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_2_1_double_ip_batch_10_istride_2_1_1_HI_ostride_4_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_2_1_1_double_ip_batch_1_istride_1_1_1_HI_ostride_2_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_2_1_1_double_ip_batch_10_istride_1_1_1_HI_ostride_2_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_1_2_double_ip_batch_1_istride_2_2_1_HI_ostride_4_4_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0"
|
|
"real_inverse_len_1_1_2_double_ip_batch_10_istride_2_2_1_HI_ostride_4_4_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0"
|
|
"real_inverse_len_8_1_2_double_ip_batch_1_istride_2_2_1_HI_ostride_4_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_8_1_2_double_ip_batch_10_istride_2_2_1_HI_ostride_4_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_2_8_double_ip_batch_1_istride_10_5_1_HI_ostride_20_10_1_R_idist_10_odist_20_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_2_8_double_ip_batch_10_istride_10_5_1_HI_ostride_20_10_1_R_idist_10_odist_20_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_8_2_double_ip_batch_1_istride_16_2_1_HI_ostride_32_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_1_8_2_double_ip_batch_10_istride_16_2_1_HI_ostride_32_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_512_7_2_double_ip_batch_10_istride_14_2_1_HI_ostride_28_4_1_R_idist_7168_odist_14336_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_7_2_512_double_ip_batch_10_istride_514_257_1_HI_ostride_1028_514_1_R_idist_3598_odist_7196_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_2_512_7_double_ip_batch_10_istride_2048_4_1_HI_ostride_4096_8_1_R_idist_4096_odist_8192_ioffset_0_0_ooffset_0_0"
|
|
// clang-format on
|
|
};
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_token_hermitian_input,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator_token(test_prob,
|
|
adhoc_hermitian_input_tokens)),
|
|
accuracy_test::TestName);
|
|
|
|
const auto adhoc_nondefault_layout_complex_tokens = {
|
|
// clang-format off
|
|
"complex_forward_len_29_4_16_double_op_batch_233_istride_540_135_5_CI_ostride_1540_140_4_CI_idist_20520_odist_55440_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_16_4_single_op_batch_253_istride_450_10_2_CI_ostride_1376_32_4_CI_idist_16650_odist_41280_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_16_4_double_ip_batch_376_istride_384_16_4_CI_ostride_384_16_4_CI_idist_10752_odist_10752_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_4_16_double_op_batch_1996_istride_1536_192_6_CI_ostride_552_24_1_CI_idist_41472_odist_17112_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_46_4_16_single_ip_batch_613_istride_441_49_1_CI_ostride_441_49_1_CI_idist_26460_odist_26460_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_46410_odist_22680_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_55250_odist_27216_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_29_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_68510_odist_34020_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_46410_odist_21600_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_55250_odist_25920_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_29_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_68510_odist_32400_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_41_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_70560_odist_70560_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_40320_odist_40320_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_19_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_33600_odist_33600_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_41_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_70560_odist_70560_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_40320_odist_40320_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_19_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_33600_odist_33600_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_8_4_single_ip_batch_1826_istride_60_6_1_CI_ostride_60_6_1_CI_idist_2340_odist_2340_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_4_8_single_ip_batch_1826_istride_60_10_1_CI_ostride_60_10_1_CI_idist_2340_odist_2340_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_4_9_single_ip_batch_2450_istride_140_14_1_CI_ostride_140_14_1_CI_idist_4900_odist_4900_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_9_4_single_ip_batch_2450_istride_140_10_1_CI_ostride_140_10_1_CI_idist_4900_odist_4900_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_20_10_single_ip_batch_701_istride_2100_100_5_CI_ostride_2100_100_5_CI_idist_52500_odist_52500_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_10_20_single_ip_batch_701_istride_2100_105_5_CI_ostride_2100_105_5_CI_idist_52500_odist_52500_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_25_4_double_ip_batch_74_istride_2058_42_6_CI_ostride_2058_42_6_CI_idist_94668_odist_94668_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_4_25_double_ip_batch_74_istride_2058_294_6_CI_ostride_2058_294_6_CI_idist_94668_odist_94668_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_8_8_single_ip_batch_3646_istride_136_8_1_CI_ostride_136_8_1_CI_idist_4352_odist_4352_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_4_4_single_ip_batch_2460_istride_112_14_2_CI_ostride_112_14_2_CI_idist_2688_odist_2688_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_25_25_single_op_batch_217_istride_2610_87_3_CI_ostride_3920_112_2_CI_idist_54810_odist_188160_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_29_27_4_double_op_batch_1358_istride_216_8_1_CI_ostride_672_24_2_CI_idist_6480_odist_22176_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_29_4_27_double_op_batch_1358_istride_216_27_1_CI_ostride_672_56_2_CI_idist_6480_odist_22176_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_16_16_single_ip_batch_3_istride_342_18_1_CI_ostride_342_18_1_CI_idist_6840_odist_6840_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_19_27_27_double_op_batch_600_istride_1624_58_2_CI_ostride_3780_140_4_CI_idist_34104_odist_128520_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_81_74_single_ip_batch_2_istride_74_1_CI_ostride_74_1_CI_idist_7696_odist_7696_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_50_129_single_op_batch_3189_istride_1432_8_CI_ostride_129_1_CI_idist_81624_odist_24252_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_9_4_29_single_op_batch_3186_istride_4_1_36_CI_ostride_1_9_36_CI_idist_1044_odist_1044_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_25_4_19_single_op_batch_6953_istride_1_25_100_CI_ostride_1_475_25_CI_idist_1900_odist_1900_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_9_29_32_double_op_batch_1584_istride_1_288_9_CI_ostride_928_1_29_CI_idist_8352_odist_8352_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_9_27_19_single_op_batch_4484_istride_1_9_243_CI_ostride_27_1_243_CI_idist_4617_odist_4617_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_4_32_57_single_op_batch_567_istride_1_4_128_CI_ostride_1_228_4_CI_idist_7296_odist_7296_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_4_25_47_single_op_batch_852_istride_1_4_100_CI_ostride_1175_47_1_CI_idist_4700_odist_4700_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_9_27_23_single_op_batch_2126_istride_27_1_243_CI_ostride_621_1_27_CI_idist_5589_odist_5589_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_47_32_8_double_op_batch_445_istride_256_8_1_CI_ostride_32_1_1504_CI_idist_12032_odist_12032_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_16_29_9_single_op_batch_4808_istride_9_144_1_CI_ostride_261_1_29_CI_idist_4176_odist_4176_ioffset_0_0_ooffset_0_0"
|
|
"complex_forward_len_19_9_16_single_op_batch_4875_istride_144_16_1_CI_ostride_16_304_1_CI_idist_2736_odist_2736_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_32_38_4_double_op_batch_1202_istride_4_128_1_CI_ostride_1_32_1216_CI_idist_4864_odist_4864_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_16_9_double_op_batch_3154_istride_144_9_1_CI_ostride_1_171_19_CI_idist_2736_odist_2736_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_32_29_9_single_op_batch_3859_istride_9_288_1_CI_ostride_29_1_928_CI_idist_8352_odist_8352_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_4_23_4_single_op_batch_4948_istride_1_16_4_CI_ostride_92_4_1_CI_idist_368_odist_368_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_32_23_25_double_op_batch_2016_istride_25_800_1_CI_ostride_575_25_1_CI_idist_18400_odist_18400_ioffset_0_0",
|
|
"complex_forward_len_16_50_4_single_op_batch_441_istride_1_64_16_CI_ostride_50_1_800_CI_idist_3200_odist_3200_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_9_29_9_single_op_batch_3750_istride_1_81_9_CI_ostride_29_1_261_CI_idist_2349_odist_2349_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_8_23_8_double_op_batch_5361_istride_8_64_1_CI_ostride_184_8_1_CI_idist_1472_odist_1472_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_31_9_32_single_op_batch_3955_istride_288_1_9_CI_ostride_9_1_279_CI_idist_8928_odist_8928_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_19_9_25_double_op_batch_3123_istride_225_1_9_CI_ostride_225_25_1_CI_idist_4275_odist_4275_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_32_16_23_double_op_batch_822_istride_16_1_512_CI_ostride_368_23_1_CI_idist_11776_odist_11776_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_31_16_9_single_op_batch_2599_istride_144_9_1_CI_ostride_9_279_1_CI_idist_4464_odist_4464_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_23_16_8_double_op_batch_912_istride_186_8_1_CI_ostride_173_8_1_CI_idist_5888_odist_5888_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_23_8_25_double_ip_batch_2056_istride_323_25_1_CI_ostride_323_25_1_CI_idist_9200_odist_9200_ioffset_0_0_ooffset_0_0",
|
|
// 3D complex with slowest stride padded ({B*C+1, C, 1}) to verify a
|
|
// fused 2D kernel pinned by a min_token solution-map entry stays correct
|
|
// when reused in a 3D context with non-default strides.
|
|
"complex_forward_len_64_32_32_single_ip_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0",
|
|
"complex_forward_len_64_32_32_single_op_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0",
|
|
"complex_inverse_len_64_32_32_single_op_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0",
|
|
// clang-format on
|
|
};
|
|
|
|
const auto adhoc_nondefault_layout_real_tokens = {
|
|
// clang-format off
|
|
"real_forward_len_25_8_double_ip_batch_7851_istride_22_1_R_ostride_11_1_HI_idist_1650_odist_825_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_9_54_single_ip_batch_2976_istride_212_1_R_ostride_106_1_HI_idist_6572_odist_3286_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_81_18_double_ip_batch_2790_istride_24_1_R_ostride_12_1_HI_idist_3768_odist_1884_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_25_16_double_ip_batch_5802_istride_26_1_R_ostride_13_1_HI_idist_1300_odist_650_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_8_18_single_ip_batch_904_istride_54_1_R_ostride_27_1_HI_idist_4104_odist_2052_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_64_8_double_ip_batch_2663_istride_40_1_R_ostride_20_1_HI_idist_5560_odist_2780_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_27_16_double_ip_batch_5966_istride_94_1_R_ostride_47_1_HI_idist_4606_odist_2303_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_25_32_double_ip_batch_4876_istride_112_1_R_ostride_56_1_HI_idist_8512_odist_4256_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_9_54_single_ip_batch_2468_istride_286_1_R_ostride_143_1_HI_idist_10010_odist_5005_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_9_16_single_ip_batch_7730_istride_44_1_R_ostride_22_1_HI_idist_792_odist_396_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_8_8_double_ip_batch_4302_istride_62_1_R_ostride_31_1_HI_idist_8804_odist_4402_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_9_32_single_ip_batch_5591_istride_40_1_R_ostride_20_1_HI_idist_2320_odist_1160_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_4_1_18_double_ip_batch_2265_istride_56_28_1_R_ostride_28_14_1_HI_idist_1176_odist_588_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_20_20_single_ip_batch_4737_istride_226_1_R_ostride_113_1_HI_idist_7458_odist_3729_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_16_8_double_ip_batch_2683_istride_26_1_R_ostride_13_1_HI_idist_884_odist_442_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_16_1_18_double_ip_batch_2970_istride_196_28_1_R_ostride_98_14_1_HI_idist_3136_odist_1568_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_20_20_single_ip_batch_6342_istride_198_1_R_ostride_99_1_HI_idist_6732_odist_3366_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_8_1_8_double_ip_batch_5635_istride_250_10_1_R_ostride_125_5_1_HI_idist_2750_odist_1375_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_32_8_double_ip_batch_1752_istride_70_1_R_ostride_35_1_HI_idist_16450_odist_8225_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_81_64_single_ip_batch_7187_istride_96_1_R_ostride_48_1_HI_idist_10176_odist_5088_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_64_16_double_ip_batch_7289_istride_92_1_R_ostride_46_1_HI_idist_6256_odist_3128_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_16_16_single_ip_batch_6600_istride_38_1_R_ostride_19_1_HI_idist_1596_odist_798_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_4_18_double_ip_batch_7484_istride_128_1_R_ostride_64_1_HI_idist_16384_odist_8192_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_9_8_double_ip_batch_6908_istride_14_1_R_ostride_7_1_HI_idist_924_odist_462_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_32_18_double_ip_batch_7083_istride_30_1_R_ostride_15_1_HI_idist_1830_odist_915_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_486_double_op_batch_7014_istride_7014_R_ostride_7014_HI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_486_double_op_batch_910_istride_910_R_ostride_910_HI_idist_1_odist_1_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_15_12_76_single_ip_batch_125_istride_1280_80_1_R_ostride_640_40_1_HI_idist_32000_odist_16000_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_11_12_38_single_ip_batch_1253_istride_2128_56_1_R_ostride_1064_28_1_HI_idist_23408_odist_11704_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_36_13_46_single_ip_batch_91_istride_2340_78_1_R_ostride_1170_39_1_HI_idist_91260_odist_45630_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_20_8_38_double_ip_batch_846_istride_1392_48_1_R_ostride_696_24_1_HI_idist_66816_odist_33408_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_2_5_46_double_ip_batch_281_istride_4340_70_1_R_ostride_2170_35_1_HI_idist_30380_odist_15190_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_5_12_38_single_ip_batch_2725_istride_1012_44_1_R_ostride_506_22_1_HI_idist_17204_odist_8602_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_6_24_38_single_ip_batch_830_istride_2688_64_1_R_ostride_1344_32_1_HI_idist_48384_odist_24192_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_6_25_38_single_ip_batch_2006_istride_1360_40_1_R_ostride_680_20_1_HI_idist_13600_odist_6800_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_26_26_134_double_ip_batch_41_istride_5934_138_1_R_ostride_2967_69_1_HI_idist_231426_odist_115713_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_2_40_62_single_op_batch_922_istride_62_124_1_R_ostride_1280_32_1_HI_idist_4960_odist_2560_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_6_12_46_single_op_batch_2312_istride_46_276_1_R_ostride_288_24_1_HI_idist_3312_odist_1728_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_7_14_76_double_ip_batch_202_istride_3588_78_1_R_ostride_1794_39_1_HI_idist_150696_odist_75348_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_2_14_46_single_ip_batch_499_istride_2050_50_1_R_ostride_1025_25_1_HI_idist_71750_odist_35875_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_26_52_double_op_batch_3899_istride_240_4_R_ostride_224_7_HI_idist_15600_odist_8960_ioffset_0_0_ooffset_0_0",
|
|
"real_forward_len_160_72_72_single_op_batch_20_istride_5328_74_1_R_ostride_2664_37_1_HI_idist_852480_odist_426240_ioffset_0_0_ooffset_0_0",
|
|
"real_inverse_len_160_72_72_single_op_batch_20_istride_2664_37_1_HI_ostride_5328_74_1_R_idist_426240_odist_852480_ioffset_0_0_ooffset_0_0",
|
|
// clang-format on
|
|
};
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
adhoc_nondefault_layout_complex,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator_token(test_prob, adhoc_nondefault_layout_complex_tokens)),
|
|
accuracy_test::TestName);
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
adhoc_nondefault_layout_real,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_generator_token(test_prob, adhoc_nondefault_layout_real_tokens)),
|
|
accuracy_test::TestName);
|
|
|
|
inline auto param_even_real_odd_base_index()
|
|
{
|
|
std::vector<fft_params> params;
|
|
// C2R/R2C of even (real) lengths with odd distances (1D)
|
|
// or odd strides along first non-contiguous dimension (>1D)
|
|
|
|
const std::vector<std::vector<size_t>> test_lengths = {
|
|
// 1D
|
|
{16},
|
|
{64},
|
|
{192},
|
|
{194},
|
|
{442},
|
|
//2D
|
|
{16, 16},
|
|
{25, 16},
|
|
{64, 64},
|
|
{75, 64},
|
|
{64, 192},
|
|
{225, 192},
|
|
{128, 194},
|
|
{225, 194},
|
|
{256, 442},
|
|
{625, 442},
|
|
// 3D
|
|
{16, 16, 16},
|
|
{16, 25, 16},
|
|
{25, 16, 16},
|
|
{25, 25, 16},
|
|
{64, 64, 64},
|
|
{64, 75, 64},
|
|
{75, 64, 64},
|
|
{75, 75, 64},
|
|
{128, 128, 192},
|
|
{128, 225, 192},
|
|
{225, 128, 192},
|
|
{225, 225, 192},
|
|
{128, 128, 194},
|
|
{128, 225, 194},
|
|
{225, 128, 194},
|
|
{225, 225, 194},
|
|
};
|
|
|
|
for(auto precision : precision_range_sp_dp)
|
|
{
|
|
for(const auto& len : test_lengths)
|
|
{
|
|
fft_params param;
|
|
param.precision = precision;
|
|
param.transform_type = fft_transform_type_real_forward;
|
|
param.itype = fft_array_type_real;
|
|
param.otype = fft_array_type_hermitian_interleaved;
|
|
param.placement = fft_placement_notinplace; // cannot be in-place by definition
|
|
param.length = len;
|
|
if(len.size() == 1)
|
|
{
|
|
param.nbatch = 2;
|
|
param.istride = {1};
|
|
param.ostride = {1};
|
|
param.idist = len[0] + 1; // +1 for generating odd addresses for some rows
|
|
param.odist = len[0] / 2 + 1;
|
|
}
|
|
else if(len.size() > 1)
|
|
{
|
|
param.nbatch = 1;
|
|
param.idist = 0; // irrelevant for batch size 1
|
|
param.odist = 0; // irrelevant for batch size 1
|
|
param.istride.resize(len.size());
|
|
param.ostride.resize(len.size());
|
|
// unit stride along fastest-varying dimension
|
|
param.istride.back() = param.ostride.back() = 1;
|
|
param.istride[len.size() - 2]
|
|
= len.back() + 1; // +1 for generating odd addresses for some rows
|
|
param.ostride[len.size() - 2] = len.back() / 2 + 1;
|
|
for(auto dim = len.size() - 2; dim-- > 0;)
|
|
{
|
|
param.istride[dim] = param.istride[dim + 1] * len[dim + 1];
|
|
param.ostride[dim] = param.ostride[dim + 1] * len[dim + 1];
|
|
}
|
|
}
|
|
param.validate();
|
|
|
|
{
|
|
const double roll = hash_prob(random_seed, param.token());
|
|
const double run_prob
|
|
= test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0)
|
|
* (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0)
|
|
* (param.is_real() ? real_prob_factor : 1.0)
|
|
* (param.is_callback() ? callback_prob_factor : 1.0);
|
|
|
|
if(roll > run_prob)
|
|
{
|
|
if(verbose > 4)
|
|
{
|
|
std::cout << "Test skipped (probability " << run_prob << " > " << roll
|
|
<< ")\n";
|
|
}
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
if(param.valid(0))
|
|
{
|
|
params.push_back(param);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return params;
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(adhoc_real_even_length_odd_base_index,
|
|
accuracy_test,
|
|
::testing::ValuesIn(param_even_real_odd_base_index()),
|
|
accuracy_test::TestName);
|