//===----------------------------------------------------------------------===// // // Part of libcu++, the C++ Standard Library for your entire system, // under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. // //===----------------------------------------------------------------------===// // MIT License // // Modifications Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in all // copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. #ifndef _CUDA___BIT_BITFILED_INSERT_EXTRACT_H #define _CUDA___BIT_BITFILED_INSERT_EXTRACT_H #include #if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) # pragma GCC system_header #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) # pragma clang system_header #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header #include #include #include #include #include #include #include _LIBCUDACXX_BEGIN_NAMESPACE_CUDA #if __cccl_ptx_isa >= 200 _CCCL_NODISCARD _CCCL_HIDE_FROM_ABI _CCCL_DEVICE uint32_t __bfi(uint32_t __dest, uint32_t __source, int __start, int __width) noexcept { asm("bfi.b32 %0, %1, %2, %3, %4;" : "=r"(__dest) : "r"(__source), "r"(__dest), "r"(__start), "r"(__width)); return __dest; } _CCCL_NODISCARD _CCCL_HIDE_FROM_ABI _CCCL_DEVICE uint64_t __bfi(uint64_t __dest, uint64_t __source, int __start, int __width) noexcept { asm("bfi.b64 %0, %1, %2, %3, %4;" : "=l"(__dest) : "l"(__source), "l"(__dest), "r"(__start), "r"(__width)); return __dest; } _CCCL_NODISCARD _CCCL_HIDE_FROM_ABI _CCCL_DEVICE uint32_t __bfe(uint32_t __value, int __start, int __width) noexcept { // uint32_t __ret; // asm("bfe.u32 %0, %1, %2, %3;" : "=r"(__ret) : "r"(__value), "r"(__start), "r"(__width)); // return __ret; // TODO(HIP/AMD): this builtin seems to not work correctly/as intended here atm return __builtin_amdgcn_ubfe(__value, __start, __width); } _CCCL_NODISCARD _CCCL_HIDE_FROM_ABI _CCCL_DEVICE uint64_t __bfe(uint64_t __value, int __start, int __width) noexcept { // uint64_t __ret; // asm("bfe.u64 %0, %1, %2, %3;" : "=l"(__ret) : "l"(__value), "r"(__start), "r"(__width)); // return __ret; // TODO(HIP/AMD): this builtin seems to not work correctly/as intended here atm return __builtin_amdgcn_ubfe(__value, __start, __width); } #endif // __cccl_ptx_isa >= 200 template _CCCL_NODISCARD _LIBCUDACXX_HIDE_FROM_ABI constexpr _Tp bitfield_insert(const _Tp __dest, const _Tp __source, int __start, int __width) noexcept { static_assert(_CUDA_VSTD::__cccl_is_unsigned_integer_v<_Tp>, "bitfield_insert() requires unsigned integer types"); [[maybe_unused]] constexpr auto __digits = _CUDA_VSTD::numeric_limits<_Tp>::digits; _CCCL_ASSERT(__width >= 0 && __width <= __digits, "width out of range"); _CCCL_ASSERT(__start >= 0 && __start <= __digits, "start position out of range"); _CCCL_ASSERT(__start + __width <= __digits, "start position + width out of range"); #ifndef __HIP_PLATFORM_AMD__ if constexpr (sizeof(_Tp) <= sizeof(uint64_t)) { if (!_CUDA_VSTD::__cccl_default_is_constant_evaluated()) { // clang-format off NV_DISPATCH_TARGET( // all SM < 70 NV_PROVIDES_SM_70, (;), NV_IS_DEVICE_LIBHIPCXX, (using _Up = _CUDA_VSTD::_If; return ::cuda::__bfi(static_cast<_Up>(__dest), static_cast<_Up>(__source), __start, __width);)) // clang-format on } } #endif auto __mask = ::cuda::bitmask<_Tp>(__start, __width); return (::cuda::__shl(__source, __start) & __mask) | (__dest & ~__mask); } template _CCCL_NODISCARD _LIBCUDACXX_HIDE_FROM_ABI constexpr _Tp bitfield_extract(const _Tp __value, int __start, int __width) noexcept { static_assert(_CUDA_VSTD::__cccl_is_unsigned_integer_v<_Tp>, "bitfield_extract() requires unsigned integer types"); [[maybe_unused]] constexpr auto __digits = _CUDA_VSTD::numeric_limits<_Tp>::digits; _CCCL_ASSERT(__width >= 0 && __width <= __digits, "width out of range"); _CCCL_ASSERT(__start >= 0 && __start <= __digits, "start position out of range"); _CCCL_ASSERT(__start + __width <= __digits, "start position + width out of range"); #ifndef __HIP_PLATFORM_AMD__ if constexpr (sizeof(_Tp) <= sizeof(uint32_t)) { if (!_CUDA_VSTD::__cccl_default_is_constant_evaluated()) { // clang-format off NV_DISPATCH_TARGET( // all SM < 70 NV_PROVIDES_SM_70, (;), NV_IS_DEVICE_LIBHIPCXX, (using _Up = _CUDA_VSTD::_If; return ::cuda::__bfe(static_cast<_Up>(__value), __start, __width);)) // clang-format on } } #endif return ::cuda::__shr(__value, __start) & ::cuda::bitmask<_Tp>(0, __width); } _LIBCUDACXX_END_NAMESPACE_CUDA #endif // _CUDA___BIT_BITFILED_INSERT_EXTRACT_H