// Copyright (C) 2021 - 2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. #include "../../shared/accuracy_test.h" #include "../../shared/params_gen.h" std::vector> adhoc_sizes = { // sizes that exercise L1D_TRTRT subplan of 2D_RTRT or 3D_TRTRTR {1, 220}, {1, 330}, {81, 220, 36}, // L1D_CC subplan of 3D_TRTRTR {4, 4, 8192}, // SBRC 192 with special param {192, 192, 192}, {192, 84, 84}, // Failure with build_CS_3D_BLOCK_RC {680, 128, 128}, // Large 1D primes that fall above the block threshold (length 262144). // Bluestein requires two forwards and one inverse FFTs, and the plan // for these sizes breakdown these FFTs either as: // L1D_TRTRT (T + STOCKHAM + T + STOCKHAM + T) for lengthBlue <= 4096^2 // or // L1D_TRTRT (T + L1D_CC + STOCKHAM_BL_CC + STOCHMAM_BL_RC + T + STOCKHAM + T) // for lengthBlue > 4096^2. {196597}, {25165813}, // 2D single-kernel bluestein size combined with multi-kernel bluestein {19, 2053}, // TILE_UNALIGNED type of SBRC 3D ERC {98, 98, 98}, // 3D_BLOCK_CR {336, 336, 56}, // real-Stockham fusion boundary cases. // 768, 1536, 3072 newly fuse on more LDS at sp/dp. // 4096, 8192 verify the LDS-fit helper correctly refuses too-large fused // kernels (negative test). param_generator covers complex_forward and // real_forward for each entry, and the round-trip in accuracy_test.h // exercises C2R automatically. {768}, {1536}, {3072}, {4096}, {8192}, }; const static std::vector> stride_range = {{1}}; static std::vector> ioffset_range_zero = {{0, 0}}; static std::vector> ooffset_range_zero = {{0, 0}}; static std::vector> ioffset_range = {{0, 0}, {1, 1}}; static std::vector> ooffset_range = {{0, 0}, {1, 1}}; INSTANTIATE_TEST_SUITE_P(adhoc, accuracy_test, ::testing::ValuesIn(param_generator(test_prob, adhoc_sizes, precision_range_sp_dp, batch_range, stride_range, stride_range, ioffset_range_zero, ooffset_range_zero, place_range, true)), accuracy_test::TestName); INSTANTIATE_TEST_SUITE_P(DISABLED_offset_adhoc, accuracy_test, ::testing::ValuesIn(param_generator(test_prob, adhoc_sizes, precision_range_full, batch_range, stride_range, stride_range, ioffset_range, ooffset_range, place_range, true)), accuracy_test::TestName); // Test that dist is ignored for batch-1 transforms. Normally, // in-place transforms require same dist, but for batch-1 dist isn't // used for anything and differing dist should be allowed. inline auto param_permissive_iodist() { std::vector> lengths = adhoc_sizes; lengths.push_back({4}); std::vector params; for(const auto precision : precision_range_sp_dp) { for(const auto trans_type : trans_type_range) { for(const auto& types : generate_types(trans_type, place_range, true)) { if(std::get<1>(types) != fft_placement_inplace) continue; for(const auto& len : lengths) { fft_params param; param.length = len; param.precision = precision; param.idist = 2; param.odist = 3; param.transform_type = std::get<0>(types); param.placement = std::get<1>(types); param.itype = std::get<2>(types); param.otype = std::get<3>(types); param.validate(); const double roll = hash_prob(random_seed, param.token()); const double run_prob = test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0) * (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0) * (param.is_real() ? real_prob_factor : 1.0) * (param.is_callback() ? callback_prob_factor : 1.0); if(roll > run_prob) { if(verbose > 4) { std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n"; } continue; } if(param.valid(0)) { params.push_back(param); } } } } } return params; } INSTANTIATE_TEST_SUITE_P(adhoc_dist, accuracy_test, ::testing::ValuesIn(param_permissive_iodist()), accuracy_test::TestName); inline auto param_adhoc_colmajor() { // generate basic FFTs of adhoc sizes auto params = param_generator(test_prob, adhoc_sizes, {fft_precision_single}, {2}, stride_range, stride_range, ioffset_range_zero, ooffset_range_zero, {fft_placement_notinplace}, false); // remove any params that are: // - 1D (not enough dims to swap) // - real-complex 2D (we only get to play with higher dims, so // again not enough dims to swap) params.erase(std::remove_if(params.begin(), params.end(), [](const fft_params& param) { if(param.length.size() == 1) return true; if(param.length.size() == 2) { if(param.transform_type == fft_transform_type_real_forward || param.transform_type == fft_transform_type_real_inverse) return true; } return false; }), params.end()); // reverse length/stride order on remaining params to make them // col-major std::for_each(params.begin(), params.end(), [](fft_params& param) { size_t start_dim = 0; // for real-complex we can't touch the fastest dim if(param.transform_type == fft_transform_type_real_forward || param.transform_type == fft_transform_type_real_inverse) ++start_dim; std::reverse(param.length.rbegin() + start_dim, param.length.rend()); std::reverse(param.istride.rbegin() + start_dim, param.istride.rend()); std::reverse(param.ostride.rbegin() + start_dim, param.ostride.rend()); }); return params; } INSTANTIATE_TEST_SUITE_P(adhoc_colmajor, accuracy_test, ::testing::ValuesIn(param_adhoc_colmajor()), accuracy_test::TestName); inline auto param_adhoc_stride() { std::vector params; for(const auto precision : precision_range_full) { for(const auto& types : generate_types(fft_transform_type_complex_forward, {fft_placement_inplace, fft_placement_notinplace}, true)) { // 2D with non-contiguous strides and dist fft_params param; param.length = {2, 35}; param.precision = precision; param.idist = 200; param.odist = 200; param.transform_type = fft_transform_type_complex_forward; param.nbatch = 2; param.placement = std::get<1>(types); param.itype = std::get<2>(types); param.otype = std::get<3>(types); param.istride = {90, 2}; param.ostride = {90, 2}; param.validate(); const double roll = hash_prob(random_seed, param.token()); const double run_prob = test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0) * (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0) * (param.is_real() ? real_prob_factor : 1.0) * (param.is_callback() ? callback_prob_factor : 1.0); if(roll > run_prob) { if(verbose > 4) { std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n"; } continue; } else { if(param.valid(0)) { params.push_back(param); } } } // test C2R/R2C with non-contiguous higher strides and dist - we // want unit stride for length0 so we do the even-length optimization for(const auto& types : generate_types(fft_transform_type_real_forward, {fft_placement_notinplace}, true)) { fft_params param; param.length = {4, 4, 4}; param.precision = precision; param.idist = 0; param.odist = 0; param.transform_type = fft_transform_type_real_forward; param.nbatch = 2; param.placement = std::get<1>(types); param.itype = std::get<2>(types); param.otype = std::get<3>(types); param.istride = {16, 4, 1}; param.ostride = {16, 4, 1}; param.validate(); { const double roll = hash_prob(random_seed, param.token()); const double run_prob = test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0) * (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0) * (param.is_real() ? real_prob_factor : 1.0) * (param.is_callback() ? callback_prob_factor : 1.0); if(roll > run_prob) { if(verbose > 4) { std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n"; } continue; } else { if(param.valid(0)) { params.push_back(param); } } } param.length = {2, 2, 2}; param.precision = precision; param.idist = 0; param.odist = 0; param.transform_type = fft_transform_type_real_forward; param.nbatch = 2; param.placement = std::get<1>(types); param.itype = std::get<2>(types); param.otype = std::get<3>(types); param.istride = {20, 6, 1}; param.ostride = {20, 6, 1}; param.validate(); { const double roll = hash_prob(random_seed, param.token()); const double run_prob = test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0) * (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0) * (param.is_real() ? real_prob_factor : 1.0) * (param.is_callback() ? callback_prob_factor : 1.0); if(roll > run_prob) { if(verbose > 4) { std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n"; } continue; } else { if(param.valid(0)) { params.push_back(param); } } } } } return params; } INSTANTIATE_TEST_SUITE_P(adhoc_stride, accuracy_test, ::testing::ValuesIn(param_adhoc_stride()), accuracy_test::TestName); const auto adhoc_tokens = { // clang-format off "complex_forward_len_4_4_4_single_op_batch_2_istride_16_4_1_CI_ostride_4_16_1_CI_idist_64_odist_64_ioffset_0_0_ooffset_0_0", "complex_forward_len_512_64_single_ip_batch_3_istride_192_3_CI_ostride_192_3_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "real_forward_len_1024_1024_1024_single_op_batch_1_istride_1048576_1024_1_R_ostride_525312_513_1_HI_idist_1073741824_odist_537919488_ioffset_0_0_ooffset_0_0", "complex_forward_len_6144_single_ip_batch_34_istride_35_CI_ostride_35_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "real_forward_len_8192_single_ip_batch_65537_istride_1_R_ostride_1_HI_idist_8194_odist_4097_ioffset_0_0_ooffset_0_0", "real_forward_len_520_single_op_batch_270400_istride_1_R_ostride_1_HI_idist_520_odist_261_ioffset_0_0_ooffset_0_0", "real_forward_len_630_single_op_batch_396900_istride_1_R_ostride_1_HI_idist_630_odist_316_ioffset_0_0_ooffset_0_0", "real_forward_len_660_single_op_batch_435600_istride_1_R_ostride_1_HI_idist_660_odist_331_ioffset_0_0_ooffset_0_0", "real_forward_len_700_single_op_batch_490000_istride_1_R_ostride_1_HI_idist_700_odist_351_ioffset_0_0_ooffset_0_0", "real_forward_len_728_single_op_batch_529984_istride_1_R_ostride_1_HI_idist_728_odist_365_ioffset_0_0_ooffset_0_0", "real_forward_len_968_single_op_batch_937024_istride_1_R_ostride_1_HI_idist_968_odist_485_ioffset_0_0_ooffset_0_0", "real_forward_len_1020_single_op_batch_1040400_istride_1_R_ostride_1_HI_idist_1020_odist_511_ioffset_0_0_ooffset_0_0", "real_forward_len_378_42_single_ip_batch_66000_istride_44_1_R_ostride_22_1_HI_idist_16632_odist_8316_ioffset_0_0_ooffset_0_0", "real_forward_len_527_25_single_ip_batch_67500_istride_26_1_R_ostride_13_1_HI_idist_13702_odist_6851_ioffset_0_0_ooffset_0_0", "real_forward_len_630_38_single_ip_batch_65540_istride_40_1_R_ostride_20_1_HI_idist_25200_odist_12600_ioffset_0_0_ooffset_0_0", // degenerate length-1 test cases "complex_forward_len_1_single_ip_batch_2_istride_1_CI_ostride_1_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "complex_forward_len_1_single_op_batch_2_istride_1_CI_ostride_1_CI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "real_forward_len_2_single_ip_batch_1_istride_1_R_ostride_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0", "real_forward_len_2_single_op_batch_1_istride_1_R_ostride_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0", "real_forward_len_1_2_single_ip_batch_2_istride_1_1_R_ostride_1_1_HI_idist_4_odist_2_ioffset_0_0_ooffset_0_0", "real_forward_len_1_2_single_op_batch_2_istride_1_1_R_ostride_1_1_HI_idist_2_odist_2_ioffset_0_0_ooffset_0_0", // clang-format on }; INSTANTIATE_TEST_SUITE_P(adhoc_token, accuracy_test, ::testing::ValuesIn(param_generator_token(test_prob, adhoc_tokens)), accuracy_test::TestName); const auto adhoc_hermitian_input_tokens = { // clang-format off "real_inverse_len_1_8_double_ip_batch_1_istride_5_1_HI_ostride_10_1_R_idist_5_odist_10_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_8_double_ip_batch_10_istride_5_1_HI_ostride_10_1_R_idist_5_odist_10_ioffset_0_0_ooffset_0_0", "real_inverse_len_8_1_double_ip_batch_1_istride_1_1_HI_ostride_2_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0", "real_inverse_len_8_1_double_ip_batch_10_istride_1_1_HI_ostride_2_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0", "real_inverse_len_2_7_double_ip_batch_1_istride_4_1_HI_ostride_8_1_R_idist_8_odist_16_ioffset_0_0_ooffset_0_0", "real_inverse_len_7_2_double_ip_batch_1_istride_2_1_HI_ostride_4_1_R_idist_14_odist_28_ioffset_0_0_ooffset_0_0", "real_inverse_len_16_512_double_ip_batch_1_istride_257_1_HI_ostride_514_1_R_idist_4112_odist_8224_ioffset_0_0_ooffset_0_0", "real_inverse_len_16_512_double_ip_batch_10_istride_257_1_HI_ostride_514_1_R_idist_4112_odist_8224_ioffset_0_0_ooffset_0_0", "real_inverse_len_512_16_double_ip_batch_1_istride_9_1_HI_ostride_18_1_R_idist_4608_odist_9216_ioffset_0_0_ooffset_0_0", "real_inverse_len_512_16_double_ip_batch_10_istride_9_1_HI_ostride_18_1_R_idist_4608_odist_9216_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_1_1_double_ip_batch_1_istride_1_1_1_HI_ostride_2_2_1_R_idist_1_odist_2_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_1_1_double_ip_batch_10_istride_1_1_1_HI_ostride_2_2_1_R_idist_1_odist_2_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_2_1_double_ip_batch_1_istride_2_1_1_HI_ostride_4_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_2_1_double_ip_batch_10_istride_2_1_1_HI_ostride_4_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0", "real_inverse_len_2_1_1_double_ip_batch_1_istride_1_1_1_HI_ostride_2_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0", "real_inverse_len_2_1_1_double_ip_batch_10_istride_1_1_1_HI_ostride_2_2_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_1_2_double_ip_batch_1_istride_2_2_1_HI_ostride_4_4_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0" "real_inverse_len_1_1_2_double_ip_batch_10_istride_2_2_1_HI_ostride_4_4_1_R_idist_2_odist_4_ioffset_0_0_ooffset_0_0" "real_inverse_len_8_1_2_double_ip_batch_1_istride_2_2_1_HI_ostride_4_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0", "real_inverse_len_8_1_2_double_ip_batch_10_istride_2_2_1_HI_ostride_4_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_2_8_double_ip_batch_1_istride_10_5_1_HI_ostride_20_10_1_R_idist_10_odist_20_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_2_8_double_ip_batch_10_istride_10_5_1_HI_ostride_20_10_1_R_idist_10_odist_20_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_8_2_double_ip_batch_1_istride_16_2_1_HI_ostride_32_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0", "real_inverse_len_1_8_2_double_ip_batch_10_istride_16_2_1_HI_ostride_32_4_1_R_idist_16_odist_32_ioffset_0_0_ooffset_0_0", "real_inverse_len_512_7_2_double_ip_batch_10_istride_14_2_1_HI_ostride_28_4_1_R_idist_7168_odist_14336_ioffset_0_0_ooffset_0_0", "real_inverse_len_7_2_512_double_ip_batch_10_istride_514_257_1_HI_ostride_1028_514_1_R_idist_3598_odist_7196_ioffset_0_0_ooffset_0_0", "real_inverse_len_2_512_7_double_ip_batch_10_istride_2048_4_1_HI_ostride_4096_8_1_R_idist_4096_odist_8192_ioffset_0_0_ooffset_0_0" // clang-format on }; INSTANTIATE_TEST_SUITE_P(adhoc_token_hermitian_input, accuracy_test, ::testing::ValuesIn(param_generator_token(test_prob, adhoc_hermitian_input_tokens)), accuracy_test::TestName); const auto adhoc_nondefault_layout_complex_tokens = { // clang-format off "complex_forward_len_29_4_16_double_op_batch_233_istride_540_135_5_CI_ostride_1540_140_4_CI_idist_20520_odist_55440_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_16_4_single_op_batch_253_istride_450_10_2_CI_ostride_1376_32_4_CI_idist_16650_odist_41280_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_16_4_double_ip_batch_376_istride_384_16_4_CI_ostride_384_16_4_CI_idist_10752_odist_10752_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_4_16_double_op_batch_1996_istride_1536_192_6_CI_ostride_552_24_1_CI_idist_41472_odist_17112_ioffset_0_0_ooffset_0_0", "complex_forward_len_46_4_16_single_ip_batch_613_istride_441_49_1_CI_ostride_441_49_1_CI_idist_26460_odist_26460_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_46410_odist_22680_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_55250_odist_27216_ioffset_0_0_ooffset_0_0", "complex_forward_len_29_25_16_single_op_batch_3261_istride_2210_85_5_CI_ostride_1134_42_2_CI_idist_68510_odist_34020_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_46410_odist_21600_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_55250_odist_25920_ioffset_0_0_ooffset_0_0", "complex_forward_len_29_16_25_single_op_batch_3261_istride_2210_130_5_CI_ostride_1080_60_2_CI_idist_68510_odist_32400_ioffset_0_0_ooffset_0_0", "complex_inverse_len_41_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_70560_odist_70560_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_40320_odist_40320_ioffset_0_0_ooffset_0_0", "complex_inverse_len_19_9_8_double_ip_batch_160_istride_1680_56_4_CI_ostride_1680_56_4_CI_idist_33600_odist_33600_ioffset_0_0_ooffset_0_0", "complex_inverse_len_41_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_70560_odist_70560_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_40320_odist_40320_ioffset_0_0_ooffset_0_0", "complex_inverse_len_19_8_9_double_ip_batch_160_istride_1680_120_4_CI_ostride_1680_120_4_CI_idist_33600_odist_33600_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_8_4_single_ip_batch_1826_istride_60_6_1_CI_ostride_60_6_1_CI_idist_2340_odist_2340_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_4_8_single_ip_batch_1826_istride_60_10_1_CI_ostride_60_10_1_CI_idist_2340_odist_2340_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_4_9_single_ip_batch_2450_istride_140_14_1_CI_ostride_140_14_1_CI_idist_4900_odist_4900_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_9_4_single_ip_batch_2450_istride_140_10_1_CI_ostride_140_10_1_CI_idist_4900_odist_4900_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_20_10_single_ip_batch_701_istride_2100_100_5_CI_ostride_2100_100_5_CI_idist_52500_odist_52500_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_10_20_single_ip_batch_701_istride_2100_105_5_CI_ostride_2100_105_5_CI_idist_52500_odist_52500_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_25_4_double_ip_batch_74_istride_2058_42_6_CI_ostride_2058_42_6_CI_idist_94668_odist_94668_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_4_25_double_ip_batch_74_istride_2058_294_6_CI_ostride_2058_294_6_CI_idist_94668_odist_94668_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_8_8_single_ip_batch_3646_istride_136_8_1_CI_ostride_136_8_1_CI_idist_4352_odist_4352_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_4_4_single_ip_batch_2460_istride_112_14_2_CI_ostride_112_14_2_CI_idist_2688_odist_2688_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_25_25_single_op_batch_217_istride_2610_87_3_CI_ostride_3920_112_2_CI_idist_54810_odist_188160_ioffset_0_0_ooffset_0_0", "complex_inverse_len_29_27_4_double_op_batch_1358_istride_216_8_1_CI_ostride_672_24_2_CI_idist_6480_odist_22176_ioffset_0_0_ooffset_0_0", "complex_inverse_len_29_4_27_double_op_batch_1358_istride_216_27_1_CI_ostride_672_56_2_CI_idist_6480_odist_22176_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_16_16_single_ip_batch_3_istride_342_18_1_CI_ostride_342_18_1_CI_idist_6840_odist_6840_ioffset_0_0_ooffset_0_0", "complex_inverse_len_19_27_27_double_op_batch_600_istride_1624_58_2_CI_ostride_3780_140_4_CI_idist_34104_odist_128520_ioffset_0_0_ooffset_0_0", "complex_forward_len_81_74_single_ip_batch_2_istride_74_1_CI_ostride_74_1_CI_idist_7696_odist_7696_ioffset_0_0_ooffset_0_0", "complex_forward_len_50_129_single_op_batch_3189_istride_1432_8_CI_ostride_129_1_CI_idist_81624_odist_24252_ioffset_0_0_ooffset_0_0", "complex_inverse_len_9_4_29_single_op_batch_3186_istride_4_1_36_CI_ostride_1_9_36_CI_idist_1044_odist_1044_ioffset_0_0_ooffset_0_0", "complex_inverse_len_25_4_19_single_op_batch_6953_istride_1_25_100_CI_ostride_1_475_25_CI_idist_1900_odist_1900_ioffset_0_0_ooffset_0_0", "complex_forward_len_9_29_32_double_op_batch_1584_istride_1_288_9_CI_ostride_928_1_29_CI_idist_8352_odist_8352_ioffset_0_0_ooffset_0_0", "complex_inverse_len_9_27_19_single_op_batch_4484_istride_1_9_243_CI_ostride_27_1_243_CI_idist_4617_odist_4617_ioffset_0_0_ooffset_0_0", "complex_forward_len_4_32_57_single_op_batch_567_istride_1_4_128_CI_ostride_1_228_4_CI_idist_7296_odist_7296_ioffset_0_0_ooffset_0_0", "complex_forward_len_4_25_47_single_op_batch_852_istride_1_4_100_CI_ostride_1175_47_1_CI_idist_4700_odist_4700_ioffset_0_0_ooffset_0_0", "complex_inverse_len_9_27_23_single_op_batch_2126_istride_27_1_243_CI_ostride_621_1_27_CI_idist_5589_odist_5589_ioffset_0_0_ooffset_0_0", "complex_forward_len_47_32_8_double_op_batch_445_istride_256_8_1_CI_ostride_32_1_1504_CI_idist_12032_odist_12032_ioffset_0_0_ooffset_0_0", "complex_forward_len_16_29_9_single_op_batch_4808_istride_9_144_1_CI_ostride_261_1_29_CI_idist_4176_odist_4176_ioffset_0_0_ooffset_0_0" "complex_forward_len_19_9_16_single_op_batch_4875_istride_144_16_1_CI_ostride_16_304_1_CI_idist_2736_odist_2736_ioffset_0_0_ooffset_0_0", "complex_forward_len_32_38_4_double_op_batch_1202_istride_4_128_1_CI_ostride_1_32_1216_CI_idist_4864_odist_4864_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_16_9_double_op_batch_3154_istride_144_9_1_CI_ostride_1_171_19_CI_idist_2736_odist_2736_ioffset_0_0_ooffset_0_0", "complex_forward_len_32_29_9_single_op_batch_3859_istride_9_288_1_CI_ostride_29_1_928_CI_idist_8352_odist_8352_ioffset_0_0_ooffset_0_0", "complex_inverse_len_4_23_4_single_op_batch_4948_istride_1_16_4_CI_ostride_92_4_1_CI_idist_368_odist_368_ioffset_0_0_ooffset_0_0", "complex_forward_len_32_23_25_double_op_batch_2016_istride_25_800_1_CI_ostride_575_25_1_CI_idist_18400_odist_18400_ioffset_0_0", "complex_forward_len_16_50_4_single_op_batch_441_istride_1_64_16_CI_ostride_50_1_800_CI_idist_3200_odist_3200_ioffset_0_0_ooffset_0_0", "complex_forward_len_9_29_9_single_op_batch_3750_istride_1_81_9_CI_ostride_29_1_261_CI_idist_2349_odist_2349_ioffset_0_0_ooffset_0_0", "complex_inverse_len_8_23_8_double_op_batch_5361_istride_8_64_1_CI_ostride_184_8_1_CI_idist_1472_odist_1472_ioffset_0_0_ooffset_0_0", "complex_inverse_len_31_9_32_single_op_batch_3955_istride_288_1_9_CI_ostride_9_1_279_CI_idist_8928_odist_8928_ioffset_0_0_ooffset_0_0", "complex_forward_len_19_9_25_double_op_batch_3123_istride_225_1_9_CI_ostride_225_25_1_CI_idist_4275_odist_4275_ioffset_0_0_ooffset_0_0", "complex_inverse_len_32_16_23_double_op_batch_822_istride_16_1_512_CI_ostride_368_23_1_CI_idist_11776_odist_11776_ioffset_0_0_ooffset_0_0", "complex_forward_len_31_16_9_single_op_batch_2599_istride_144_9_1_CI_ostride_9_279_1_CI_idist_4464_odist_4464_ioffset_0_0_ooffset_0_0", "complex_inverse_len_23_16_8_double_op_batch_912_istride_186_8_1_CI_ostride_173_8_1_CI_idist_5888_odist_5888_ioffset_0_0_ooffset_0_0", "complex_forward_len_23_8_25_double_ip_batch_2056_istride_323_25_1_CI_ostride_323_25_1_CI_idist_9200_odist_9200_ioffset_0_0_ooffset_0_0", // 3D complex with slowest stride padded ({B*C+1, C, 1}) to verify a // fused 2D kernel pinned by a min_token solution-map entry stays correct // when reused in a 3D context with non-default strides. "complex_forward_len_64_32_32_single_ip_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0", "complex_forward_len_64_32_32_single_op_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0", "complex_inverse_len_64_32_32_single_op_batch_2_istride_1025_32_1_CI_ostride_1025_32_1_CI_idist_65600_odist_65600_ioffset_0_0_ooffset_0_0", // clang-format on }; const auto adhoc_nondefault_layout_real_tokens = { // clang-format off "real_forward_len_25_8_double_ip_batch_7851_istride_22_1_R_ostride_11_1_HI_idist_1650_odist_825_ioffset_0_0_ooffset_0_0", "real_forward_len_9_54_single_ip_batch_2976_istride_212_1_R_ostride_106_1_HI_idist_6572_odist_3286_ioffset_0_0_ooffset_0_0", "real_forward_len_81_18_double_ip_batch_2790_istride_24_1_R_ostride_12_1_HI_idist_3768_odist_1884_ioffset_0_0_ooffset_0_0", "real_forward_len_25_16_double_ip_batch_5802_istride_26_1_R_ostride_13_1_HI_idist_1300_odist_650_ioffset_0_0_ooffset_0_0", "real_forward_len_8_18_single_ip_batch_904_istride_54_1_R_ostride_27_1_HI_idist_4104_odist_2052_ioffset_0_0_ooffset_0_0", "real_forward_len_64_8_double_ip_batch_2663_istride_40_1_R_ostride_20_1_HI_idist_5560_odist_2780_ioffset_0_0_ooffset_0_0", "real_forward_len_27_16_double_ip_batch_5966_istride_94_1_R_ostride_47_1_HI_idist_4606_odist_2303_ioffset_0_0_ooffset_0_0", "real_forward_len_25_32_double_ip_batch_4876_istride_112_1_R_ostride_56_1_HI_idist_8512_odist_4256_ioffset_0_0_ooffset_0_0", "real_forward_len_9_54_single_ip_batch_2468_istride_286_1_R_ostride_143_1_HI_idist_10010_odist_5005_ioffset_0_0_ooffset_0_0", "real_forward_len_9_16_single_ip_batch_7730_istride_44_1_R_ostride_22_1_HI_idist_792_odist_396_ioffset_0_0_ooffset_0_0", "real_forward_len_8_8_double_ip_batch_4302_istride_62_1_R_ostride_31_1_HI_idist_8804_odist_4402_ioffset_0_0_ooffset_0_0", "real_forward_len_9_32_single_ip_batch_5591_istride_40_1_R_ostride_20_1_HI_idist_2320_odist_1160_ioffset_0_0_ooffset_0_0", "real_forward_len_4_1_18_double_ip_batch_2265_istride_56_28_1_R_ostride_28_14_1_HI_idist_1176_odist_588_ioffset_0_0_ooffset_0_0", "real_forward_len_20_20_single_ip_batch_4737_istride_226_1_R_ostride_113_1_HI_idist_7458_odist_3729_ioffset_0_0_ooffset_0_0", "real_forward_len_16_8_double_ip_batch_2683_istride_26_1_R_ostride_13_1_HI_idist_884_odist_442_ioffset_0_0_ooffset_0_0", "real_forward_len_16_1_18_double_ip_batch_2970_istride_196_28_1_R_ostride_98_14_1_HI_idist_3136_odist_1568_ioffset_0_0_ooffset_0_0", "real_forward_len_20_20_single_ip_batch_6342_istride_198_1_R_ostride_99_1_HI_idist_6732_odist_3366_ioffset_0_0_ooffset_0_0", "real_forward_len_8_1_8_double_ip_batch_5635_istride_250_10_1_R_ostride_125_5_1_HI_idist_2750_odist_1375_ioffset_0_0_ooffset_0_0", "real_forward_len_32_8_double_ip_batch_1752_istride_70_1_R_ostride_35_1_HI_idist_16450_odist_8225_ioffset_0_0_ooffset_0_0", "real_forward_len_81_64_single_ip_batch_7187_istride_96_1_R_ostride_48_1_HI_idist_10176_odist_5088_ioffset_0_0_ooffset_0_0", "real_forward_len_64_16_double_ip_batch_7289_istride_92_1_R_ostride_46_1_HI_idist_6256_odist_3128_ioffset_0_0_ooffset_0_0", "real_forward_len_16_16_single_ip_batch_6600_istride_38_1_R_ostride_19_1_HI_idist_1596_odist_798_ioffset_0_0_ooffset_0_0", "real_forward_len_4_18_double_ip_batch_7484_istride_128_1_R_ostride_64_1_HI_idist_16384_odist_8192_ioffset_0_0_ooffset_0_0", "real_forward_len_9_8_double_ip_batch_6908_istride_14_1_R_ostride_7_1_HI_idist_924_odist_462_ioffset_0_0_ooffset_0_0", "real_forward_len_32_18_double_ip_batch_7083_istride_30_1_R_ostride_15_1_HI_idist_1830_odist_915_ioffset_0_0_ooffset_0_0", "real_forward_len_486_double_op_batch_7014_istride_7014_R_ostride_7014_HI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "real_forward_len_486_double_op_batch_910_istride_910_R_ostride_910_HI_idist_1_odist_1_ioffset_0_0_ooffset_0_0", "real_forward_len_15_12_76_single_ip_batch_125_istride_1280_80_1_R_ostride_640_40_1_HI_idist_32000_odist_16000_ioffset_0_0_ooffset_0_0", "real_forward_len_11_12_38_single_ip_batch_1253_istride_2128_56_1_R_ostride_1064_28_1_HI_idist_23408_odist_11704_ioffset_0_0_ooffset_0_0", "real_forward_len_36_13_46_single_ip_batch_91_istride_2340_78_1_R_ostride_1170_39_1_HI_idist_91260_odist_45630_ioffset_0_0_ooffset_0_0", "real_forward_len_20_8_38_double_ip_batch_846_istride_1392_48_1_R_ostride_696_24_1_HI_idist_66816_odist_33408_ioffset_0_0_ooffset_0_0", "real_forward_len_2_5_46_double_ip_batch_281_istride_4340_70_1_R_ostride_2170_35_1_HI_idist_30380_odist_15190_ioffset_0_0_ooffset_0_0", "real_forward_len_5_12_38_single_ip_batch_2725_istride_1012_44_1_R_ostride_506_22_1_HI_idist_17204_odist_8602_ioffset_0_0_ooffset_0_0", "real_forward_len_6_24_38_single_ip_batch_830_istride_2688_64_1_R_ostride_1344_32_1_HI_idist_48384_odist_24192_ioffset_0_0_ooffset_0_0", "real_forward_len_6_25_38_single_ip_batch_2006_istride_1360_40_1_R_ostride_680_20_1_HI_idist_13600_odist_6800_ioffset_0_0_ooffset_0_0", "real_forward_len_26_26_134_double_ip_batch_41_istride_5934_138_1_R_ostride_2967_69_1_HI_idist_231426_odist_115713_ioffset_0_0_ooffset_0_0", "real_forward_len_2_40_62_single_op_batch_922_istride_62_124_1_R_ostride_1280_32_1_HI_idist_4960_odist_2560_ioffset_0_0_ooffset_0_0", "real_forward_len_6_12_46_single_op_batch_2312_istride_46_276_1_R_ostride_288_24_1_HI_idist_3312_odist_1728_ioffset_0_0_ooffset_0_0", "real_forward_len_7_14_76_double_ip_batch_202_istride_3588_78_1_R_ostride_1794_39_1_HI_idist_150696_odist_75348_ioffset_0_0_ooffset_0_0", "real_forward_len_2_14_46_single_ip_batch_499_istride_2050_50_1_R_ostride_1025_25_1_HI_idist_71750_odist_35875_ioffset_0_0_ooffset_0_0", "real_forward_len_26_52_double_op_batch_3899_istride_240_4_R_ostride_224_7_HI_idist_15600_odist_8960_ioffset_0_0_ooffset_0_0", "real_forward_len_160_72_72_single_op_batch_20_istride_5328_74_1_R_ostride_2664_37_1_HI_idist_852480_odist_426240_ioffset_0_0_ooffset_0_0", "real_inverse_len_160_72_72_single_op_batch_20_istride_2664_37_1_HI_ostride_5328_74_1_R_idist_426240_odist_852480_ioffset_0_0_ooffset_0_0", // clang-format on }; INSTANTIATE_TEST_SUITE_P( adhoc_nondefault_layout_complex, accuracy_test, ::testing::ValuesIn(param_generator_token(test_prob, adhoc_nondefault_layout_complex_tokens)), accuracy_test::TestName); INSTANTIATE_TEST_SUITE_P( adhoc_nondefault_layout_real, accuracy_test, ::testing::ValuesIn(param_generator_token(test_prob, adhoc_nondefault_layout_real_tokens)), accuracy_test::TestName); inline auto param_even_real_odd_base_index() { std::vector params; // C2R/R2C of even (real) lengths with odd distances (1D) // or odd strides along first non-contiguous dimension (>1D) const std::vector> test_lengths = { // 1D {16}, {64}, {192}, {194}, {442}, //2D {16, 16}, {25, 16}, {64, 64}, {75, 64}, {64, 192}, {225, 192}, {128, 194}, {225, 194}, {256, 442}, {625, 442}, // 3D {16, 16, 16}, {16, 25, 16}, {25, 16, 16}, {25, 25, 16}, {64, 64, 64}, {64, 75, 64}, {75, 64, 64}, {75, 75, 64}, {128, 128, 192}, {128, 225, 192}, {225, 128, 192}, {225, 225, 192}, {128, 128, 194}, {128, 225, 194}, {225, 128, 194}, {225, 225, 194}, }; for(auto precision : precision_range_sp_dp) { for(const auto& len : test_lengths) { fft_params param; param.precision = precision; param.transform_type = fft_transform_type_real_forward; param.itype = fft_array_type_real; param.otype = fft_array_type_hermitian_interleaved; param.placement = fft_placement_notinplace; // cannot be in-place by definition param.length = len; if(len.size() == 1) { param.nbatch = 2; param.istride = {1}; param.ostride = {1}; param.idist = len[0] + 1; // +1 for generating odd addresses for some rows param.odist = len[0] / 2 + 1; } else if(len.size() > 1) { param.nbatch = 1; param.idist = 0; // irrelevant for batch size 1 param.odist = 0; // irrelevant for batch size 1 param.istride.resize(len.size()); param.ostride.resize(len.size()); // unit stride along fastest-varying dimension param.istride.back() = param.ostride.back() = 1; param.istride[len.size() - 2] = len.back() + 1; // +1 for generating odd addresses for some rows param.ostride[len.size() - 2] = len.back() / 2 + 1; for(auto dim = len.size() - 2; dim-- > 0;) { param.istride[dim] = param.istride[dim + 1] * len[dim + 1]; param.ostride[dim] = param.ostride[dim + 1] * len[dim + 1]; } } param.validate(); { const double roll = hash_prob(random_seed, param.token()); const double run_prob = test_prob * (param.is_planar() ? complex_planar_prob_factor : 1.0) * (param.is_interleaved() ? complex_interleaved_prob_factor : 1.0) * (param.is_real() ? real_prob_factor : 1.0) * (param.is_callback() ? callback_prob_factor : 1.0); if(roll > run_prob) { if(verbose > 4) { std::cout << "Test skipped (probability " << run_prob << " > " << roll << ")\n"; } continue; } else { if(param.valid(0)) { params.push_back(param); } } } } } return params; } INSTANTIATE_TEST_SUITE_P(adhoc_real_even_length_odd_base_index, accuracy_test, ::testing::ValuesIn(param_even_real_odd_base_index()), accuracy_test::TestName);