/******************************************************************************* * * MIT License * * Copyright (C) 2022-2026 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * *******************************************************************************/ #pragma once #include "hipblaslt_datatype2string.hpp" #include "hipblaslt_math.hpp" #include "hipblaslt_ostream.hpp" #include "hipblaslt_random.hpp" #include #include #include #include #include template T initialize_nan_or_zero(bool is_nan, T zero_val) { if(is_nan) { if constexpr(is_std_complex_v) { using RealT = typename T::value_type; return T(static_cast(hipblaslt_nan_rng()), static_cast(0.0)); } else { return static_cast(hipblaslt_nan_rng()); } } else { return zero_val; } } enum class ABC_dims { A, B, C }; void set_host_side_fill_kernel_state(bool enable); bool host_side_fill_kernel(); // When enabled, the hpl and trig_float device initializers emit positive-only // values. Used for ULP validation: signed zero-mean inputs make the reference // dot products cancel toward zero, where the magnitude-independent accumulation // noise floor inflates the per-element ULP error even for a correct kernel. void set_ulp_positive_init_state(bool enable); bool ulp_positive_init(); void hipblaslt_init_device(ABC_dims ABC_dims, hipblaslt_initialization init, bool is_nan, void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride, size_t batch_count, int norm_dist_one_special_type = -1); /* ============================================================================================ */ /*! \brief matrix/vector initialization: */ // for vector x (M=1, N=lengthX, lda=incx); // for complex number, the real/imag part would be initialized with the same value // Initialize matrices with random values template inline void hipblaslt_init(T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) { size_t b_idx = i_batch * stride; for(size_t j = 0; j < N; ++j) { size_t col_idx = b_idx + j * lda; if(M > 4) random_run_generator(A + col_idx, M); else { for(size_t i = 0; i < M; ++i) A[col_idx + i] = random_generator(); } } } } // Initialize matrices with random values template inline void hipblaslt_init_small( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) { size_t b_idx = i_batch * stride; for(size_t j = 0; j < N; ++j) { size_t col_idx = b_idx + j * lda; if(M > 4) random_run_generator_small(A + col_idx, M); else { for(size_t i = 0; i < M; ++i) A[col_idx + i] = random_generator_small(); } } } } // Initialize matrices with random values inline void hipblaslt_init(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init(static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_8F_UE8M0: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_32I: hipblaslt_init(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init" << std::endl; break; } } inline void hipblaslt_init_small(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_small(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_small(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_small>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_small>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_small( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_32I: hipblaslt_init_small(static_cast(A), M, N, lda, stride, batch_count); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_small" << std::endl; break; } } template inline void hipblaslt_init_sin( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) #pragma omp parallel for for(size_t j = 0; j < N; ++j) { size_t offset = j * lda + i_batch * stride; size_t offsetValue = j * M + i_batch * M * N; for(size_t i = 0; i < M; ++i) { // Real part uses sin() double val = std::sin(static_cast(i + offsetValue)); if constexpr(is_std_complex_v) { using RealT = typename T::value_type; // Imaginary part uses cos() to be distinct and non-zero. // This ensures (a+bi)*(c+di) computes full complex arithmetic. double val_imag = std::cos(static_cast(i + offsetValue)); A[i + offset] = T(static_cast(val), static_cast(val_imag)); } else { // Real/Custom Case: Direct cast of the real value A[i + offset] = static_cast(val); } } } } inline void hipblaslt_init_sin(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_sin(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_sin(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_sin>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_sin>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_sin(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_sin( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_sin not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_sin not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_sin not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_sin" << std::endl; break; } } // Initialize matrix so adjacent entries have alternating sign. // In gemm if either A or B are initialized with alternating // Checkerboard ± so first element of each row and column alternates; keeps // reduction sums from growing too large (helps 16bit with 5-bit exponent). template inline void hipblaslt_init_alternating_sign( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) #pragma omp parallel for for(size_t j = 0; j < N; ++j) { size_t offset = j * lda + i_batch * stride; for(size_t i = 0; i < M; ++i) { auto value = random_generator(); A[i + offset] = (i ^ j) & 1 ? value : negate(value); } } } inline void hipblaslt_init_alternating_sign(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_alternating_sign>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_alternating_sign>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_alternating_sign not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_alternating_sign not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_alternating_sign not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_alternating_sign" << std::endl; break; } } // Initialize matrix so adjacent entries have alternating sign. template inline void hipblaslt_init_hpl_alternating_sign( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) #pragma omp parallel for for(size_t j = 0; j < N; ++j) { size_t offset = j * lda + i_batch * stride; for(size_t i = 0; i < M; ++i) { auto value = random_hpl_generator(); A[i + offset] = (i ^ j) & 1 ? value : negate(value); } } } inline void hipblaslt_init_hpl_alternating_sign(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_hpl_alternating_sign>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_hpl_alternating_sign>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_hpl_alternating_sign( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_hpl_alternating_sign not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_hpl_alternating_sign not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_hpl_alternating_sign not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_hpl_alternating_sign" << std::endl; break; } } template inline void hipblaslt_init_cos( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) #pragma omp parallel for for(size_t j = 0; j < N; ++j) { size_t offset = j * lda + i_batch * stride; size_t offsetValue = j * M + i_batch * M * N; for(size_t i = 0; i < M; ++i) A[i + offset] = T(cos(double(i + offsetValue))); //force cast to double } } inline void hipblaslt_init_cos(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_cos(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_cos(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_cos>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_cos>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_cos(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_cos( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_cos not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_cos not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_cos not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_cos" << std::endl; break; } } // Initialize vector with HPL-like random values template inline void hipblaslt_init_hpl( std::vector& A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = random_hpl_generator(); } template inline void hipblaslt_init_hpl( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = random_hpl_generator(); } inline void hipblaslt_init_hpl(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_hpl(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_hpl(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_hpl>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_hpl>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_hpl(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_hpl( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_hpl not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_hpl not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_hpl not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_hpl" << std::endl; break; } } // Initialize vector with uniform random values in [-6, 6] template inline void hipblaslt_init_low_precision( std::vector& A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = random_low_precision_generator(); } template inline void hipblaslt_init_low_precision( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = random_low_precision_generator(); } inline void hipblaslt_init_low_precision(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_low_precision( static_cast(A), M, N, lda, stride, batch_count); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_low_precision" << std::endl; break; } } /* ============================================================================================ */ /*! \brief Initialize an array with random data, with NaN where appropriate */ template inline void hipblaslt_init_nan(T* A, size_t N) { for(size_t i = 0; i < N; ++i) A[i] = initialize_nan_or_zero(true, T{}); } template inline void hipblaslt_init_nan(T* A, size_t start_offset, size_t end_offset) { for(size_t i = start_offset; i < end_offset; ++i) A[i] = initialize_nan_or_zero(true, T{}); } inline void hipblaslt_init_nan(void* A, size_t N, hipDataType type) { switch(type) { case HIP_R_32F: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_64F: hipblaslt_init_nan(static_cast(A), N); break; case HIP_C_32F: hipblaslt_init_nan>(static_cast*>(A), N); break; case HIP_C_64F: hipblaslt_init_nan>(static_cast*>(A), N); break; case HIP_R_16F: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_16BF: hipblaslt_init_nan(static_cast(A), N); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_nan(static_cast(A), N); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_8F_E5M2: hipblaslt_init_nan(static_cast(A), N); break; #endif case HIP_R_32I: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_8I: hipblaslt_init_nan(static_cast(A), N); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_nan not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_nan not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_nan not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_nan" << std::endl; break; } } inline void hipblaslt_init_nan(void* A, size_t start_offset, size_t end_offset, hipDataType type) { switch(type) { case HIP_R_32F: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_R_64F: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_C_32F: hipblaslt_init_nan>( static_cast*>(A), start_offset, end_offset); break; case HIP_C_64F: hipblaslt_init_nan>( static_cast*>(A), start_offset, end_offset); break; case HIP_R_16F: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_R_16BF: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_nan( static_cast(A), start_offset, end_offset); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_nan( static_cast(A), start_offset, end_offset); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_R_8F_E5M2: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; #endif case HIP_R_32I: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_R_8I: hipblaslt_init_nan(static_cast(A), start_offset, end_offset); break; case HIP_R_6F_E2M3: hipblaslt_cerr << "hipblaslt_init_nan not supports FP6" << std::endl; break; case HIP_R_6F_E3M2: hipblaslt_cerr << "hipblaslt_init_nan not supports BF6" << std::endl; break; case HIP_R_4F_E2M1: hipblaslt_cerr << "hipblaslt_init_nan not supports FP4" << std::endl; break; default: hipblaslt_cerr << "Error type in hipblaslt_init_nan" << std::endl; break; } } template inline void hipblaslt_init_nan_tri( bool upper, T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { T zero_val = static_cast(0); for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) { bool is_nan_region = upper ? (j >= i) : (j <= i); T val = initialize_nan_or_zero(is_nan_region, zero_val); A[i + j * lda + i_batch * stride] = val; } } inline void hipblaslt_init_nan_tri(bool upper, void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_nan_tri(upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_nan_tri(upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_nan_tri>( upper, static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_nan_tri>( upper, static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_nan_tri(upper, static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_nan_tri( upper, static_cast(A), M, N, lda, stride, batch_count); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_nan_tri" << std::endl; break; } } template inline void hipblaslt_init_nan( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = initialize_nan_or_zero(true, T{}); } inline void hipblaslt_init_nan(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_nan(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_nan(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_nan>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_nan>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_nan(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_nan( static_cast(A), M, N, lda, stride, batch_count); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_nan" << std::endl; break; } } /* ============================================================================================ */ /*! \brief Initialize an array with random data, with Inf where appropriate */ template inline void hipblaslt_init_inf(T* A, size_t N) { for(size_t i = 0; i < N; ++i) A[i] = initialize_nan_or_zero(true, T(0.0)); } template inline void hipblaslt_init_inf(T* A, size_t start_offset, size_t end_offset) { for(size_t i = start_offset; i < end_offset; ++i) A[i] = initialize_nan_or_zero(true, T(0.0)); } inline void hipblaslt_init_inf(void* A, size_t N, hipDataType type) { switch(type) { case HIP_R_32F: hipblaslt_init_inf(static_cast(A), N); break; case HIP_R_64F: hipblaslt_init_inf(static_cast(A), N); break; case HIP_C_32F: hipblaslt_init_inf>(static_cast*>(A), N); break; case HIP_C_64F: hipblaslt_init_inf>(static_cast*>(A), N); break; case HIP_R_16F: hipblaslt_init_inf(static_cast(A), N); break; case HIP_R_16BF: hipblaslt_init_inf(static_cast(A), N); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_inf(static_cast(A), N); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_inf(static_cast(A), N); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_inf(static_cast(A), N); break; case HIP_R_8F_E5M2: hipblaslt_init_inf(static_cast(A), N); break; #endif case HIP_R_32I: hipblaslt_init_inf(static_cast(A), N); break; case HIP_R_8I: hipblaslt_init_inf(static_cast(A), N); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_inf" << std::endl; break; } } template inline void hipblaslt_init_inf( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = initialize_nan_or_zero(true, T(0.0)); } inline void hipblaslt_init_inf(void* A, size_t start_offset, size_t end_offset, hipDataType type) { switch(type) { case HIP_R_32F: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; case HIP_R_64F: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; case HIP_C_32F: hipblaslt_init_inf>( static_cast*>(A), start_offset, end_offset); break; case HIP_C_64F: hipblaslt_init_inf>( static_cast*>(A), start_offset, end_offset); break; case HIP_R_16F: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; case HIP_R_16BF: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_inf( static_cast(A), start_offset, end_offset); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_inf( static_cast(A), start_offset, end_offset); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; case HIP_R_8F_E5M2: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; #endif case HIP_R_32I: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; case HIP_R_8I: hipblaslt_init_inf(static_cast(A), start_offset, end_offset); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_inf" << std::endl; break; } } inline void hipblaslt_init_inf(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_inf(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_inf(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_inf>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_inf>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_R_16F: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_16BF: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; #if HIP_FP8_TYPE_FNUZ case HIP_R_8F_E4M3_FNUZ: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2_FNUZ: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; #endif #if HIP_FP8_TYPE_OCP case HIP_R_8F_E4M3: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8F_E5M2: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; #endif case HIP_R_32I: hipblaslt_init_inf(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_8I: hipblaslt_init_inf( static_cast(A), M, N, lda, stride, batch_count); break; default: hipblaslt_cerr << "Error type in hipblaslt_init_inf" << std::endl; break; } } /* ============================================================================================ */ /*! \brief Initialize an array with random data, with zero */ template inline void hipblaslt_init_zero( std::vector& A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = T(0); } template inline void hipblaslt_init_zero( T* A, size_t M, size_t N, size_t lda, size_t stride = 0, size_t batch_count = 1) { for(size_t i_batch = 0; i_batch < batch_count; i_batch++) for(size_t i = 0; i < M; ++i) for(size_t j = 0; j < N; ++j) A[i + j * lda + i_batch * stride] = T(0); } template inline void hipblaslt_init_zero(T* A, size_t start_offset, size_t end_offset) { for(size_t i = start_offset; i < end_offset; ++i) A[i] = T(hipblaslt_zero_rng()); } inline void hipblaslt_init_zero(void* A, size_t M, size_t N, size_t lda, hipDataType type, size_t stride = 0, size_t batch_count = 1) { switch(type) { case HIP_R_32F: hipblaslt_init_zero(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_R_64F: hipblaslt_init_zero(static_cast(A), M, N, lda, stride, batch_count); break; case HIP_C_32F: hipblaslt_init_zero>( static_cast*>(A), M, N, lda, stride, batch_count); break; case HIP_C_64F: hipblaslt_init_zero>( static_cast