/* * Copyright 2008-2013 NVIDIA Corporation * Copyright 2013 Filipe RNC Maia * Modifications Copyright© 2019-2025 Advanced Micro Devices, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include THRUST_NAMESPACE_BEGIN template THRUST_HOST_DEVICE complex::type> pow(const complex& x, const complex& y) { using T = typename ::internal::promoted_numerical_type::type; return exp(log(complex(x)) * complex(y)); } // Specialized version for complex on AMD GPUs to use FMA-based multiplication #if defined(__HIP_DEVICE_COMPILE__) && defined(__HIP_PLATFORM_AMD__) namespace detail { // FMA-aware complex multiplication for float precision on AMD GPUs. // This prevents SLP vectorizer from breaking FMA formation, which causes // numerical precision loss in complex arithmetic. // The issue occurs when vectorizer packs scalar multiplies before backend // can form FMA instructions, resulting in double rounding instead of single. THRUST_HOST_DEVICE inline thrust::complex complex_mul_fma(thrust::complex a, thrust::complex b) { // Complex multiplication: (a.r + a.i*i) * (b.r + b.i*i) // = (a.r*b.r - a.i*b.i) + (a.r*b.i + a.i*b.r)*i // Using __builtin_fmaf ensures FMA at source level: // real: a.r*b.r + (-(a.i*b.i)) = FMA(a.r, b.r, -(a.i*b.i)) // imag: a.i*b.r + a.r*b.i = FMA(a.r, b.i, a.i*b.r) float real_part = __builtin_fmaf(a.real(), b.real(), -(a.imag() * b.imag())); float imag_part = __builtin_fmaf(a.real(), b.imag(), a.imag() * b.real()); return thrust::complex(real_part, imag_part); } } // namespace detail template <> THRUST_HOST_DEVICE inline complex pow(const complex& x, const complex& y) { auto log_x = thrust::log(static_cast>(x)); auto y_log_x = detail::complex_mul_fma(static_cast>(y), log_x); return static_cast>(thrust::exp(y_log_x)); } #endif template THRUST_HOST_DEVICE complex::type> pow(const complex& x, const T1& y) { using T = typename ::internal::promoted_numerical_type::type; return exp(log(complex(x)) * T(y)); } template THRUST_HOST_DEVICE complex::type> pow(const T0& x, const complex& y) { using T = typename ::internal::promoted_numerical_type::type; #ifdef __HIP_DEVICE_COMPILE__ using ::log; #else // Find `log` by ADL. using std::log; #endif return exp(log(T(x)) * complex(y)); } THRUST_NAMESPACE_END