derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
193 lines
18 KiB
C
193 lines
18 KiB
C
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of libcu++, the C++ Standard Library for your entire system,
|
|
// under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef __LIBCUDACXX___ATOMIC_API_COMMON_H
|
|
#define __LIBCUDACXX___ATOMIC_API_COMMON_H
|
|
|
|
#include <cuda/std/detail/__config>
|
|
|
|
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
|
# pragma GCC system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
|
# pragma clang system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
|
# pragma system_header
|
|
#endif // no system header
|
|
|
|
#include <cuda/std/__atomic/types/base.h>
|
|
|
|
// API definitions for the base atomic implementation
|
|
#define _LIBCUDACXX_ATOMIC_COMMON_IMPL(_CONST, _VOLATILE) \
|
|
_CCCL_HOST_DEVICE inline bool is_lock_free() const _VOLATILE noexcept \
|
|
{ \
|
|
return _LIBCUDACXX_ATOMIC_IS_LOCK_FREE(sizeof(_Tp)); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline void store(_Tp __d, memory_order __m = memory_order_seq_cst) \
|
|
_CONST _VOLATILE noexcept _LIBCUDACXX_CHECK_STORE_MEMORY_ORDER(__m) \
|
|
{ \
|
|
__atomic_store_dispatch(&__a, __d, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp load(memory_order __m = memory_order_seq_cst) \
|
|
const _VOLATILE noexcept _LIBCUDACXX_CHECK_LOAD_MEMORY_ORDER(__m) \
|
|
{ \
|
|
return __atomic_load_dispatch(&__a, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline operator _Tp() const _VOLATILE noexcept \
|
|
{ \
|
|
return load(); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_exchange_dispatch(&__a, __d, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) \
|
|
_CONST _VOLATILE noexcept _LIBCUDACXX_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) \
|
|
{ \
|
|
return __atomic_compare_exchange_weak_dispatch(&__a, &__e, __d, __s, __f, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) \
|
|
_CONST _VOLATILE noexcept _LIBCUDACXX_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) \
|
|
{ \
|
|
return __atomic_compare_exchange_strong_dispatch(&__a, &__e, __d, __s, __f, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) \
|
|
_CONST _VOLATILE noexcept \
|
|
{ \
|
|
if (memory_order_acq_rel == __m) \
|
|
return __atomic_compare_exchange_weak_dispatch(&__a, &__e, __d, __m, memory_order_acquire, _Sco{}); \
|
|
else if (memory_order_release == __m) \
|
|
return __atomic_compare_exchange_weak_dispatch(&__a, &__e, __d, __m, memory_order_relaxed, _Sco{}); \
|
|
else \
|
|
return __atomic_compare_exchange_weak_dispatch(&__a, &__e, __d, __m, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) \
|
|
_CONST _VOLATILE noexcept \
|
|
{ \
|
|
if (memory_order_acq_rel == __m) \
|
|
return __atomic_compare_exchange_strong_dispatch(&__a, &__e, __d, __m, memory_order_acquire, _Sco{}); \
|
|
else if (memory_order_release == __m) \
|
|
return __atomic_compare_exchange_strong_dispatch(&__a, &__e, __d, __m, memory_order_relaxed, _Sco{}); \
|
|
else \
|
|
return __atomic_compare_exchange_strong_dispatch(&__a, &__e, __d, __m, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _VOLATILE noexcept \
|
|
{ \
|
|
__atomic_wait(&__a, __v, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline void notify_one() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
__atomic_notify_one(&__a, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline void notify_all() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
__atomic_notify_all(&__a, _Sco{}); \
|
|
}
|
|
|
|
// API definitions for arithmetic atomics
|
|
#define _LIBCUDACXX_ATOMIC_ARITHMETIC_IMPL(_CONST, _VOLATILE) \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_add_dispatch(&__a, __op, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_sub_dispatch(&__a, __op, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator++(int) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(_Tp(1)); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator--(int) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(_Tp(1)); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator++() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(_Tp(1)) + _Tp(1); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator--() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(_Tp(1)) - _Tp(1); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator+=(_Tp __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(__op) + __op; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator-=(_Tp __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(__op) - __op; \
|
|
}
|
|
|
|
// API definitions for bitwise atomics
|
|
#define _LIBCUDACXX_ATOMIC_BITWISE_IMPL(_CONST, _VOLATILE) \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_and_dispatch(&__a, __op, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_or_dispatch(&__a, __op, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_xor_dispatch(&__a, __op, __m, _Sco{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator&=(_Tp __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_and(__op) & __op; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator|=(_Tp __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_or(__op) | __op; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator^=(_Tp __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_xor(__op) ^ __op; \
|
|
}
|
|
|
|
// API definitions for atomics with pointers
|
|
#define _LIBCUDACXX_ATOMIC_POINTER_IMPL(_CONST, _VOLATILE) \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) \
|
|
_CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_add_dispatch(&__a, __op, __m, __thread_scope_system_tag{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) \
|
|
_CONST _VOLATILE noexcept \
|
|
{ \
|
|
return __atomic_fetch_sub_dispatch(&__a, __op, __m, __thread_scope_system_tag{}); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator++(int) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(1); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator--(int) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(1); \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator++() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(1) + 1; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator--() _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(1) - 1; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator+=(ptrdiff_t __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_add(__op) + __op; \
|
|
} \
|
|
_CCCL_HOST_DEVICE inline _Tp operator-=(ptrdiff_t __op) _CONST _VOLATILE noexcept \
|
|
{ \
|
|
return fetch_sub(__op) - __op; \
|
|
}
|
|
|
|
#endif // __LIBCUDACXX___ATOMIC_API_COMMON_H
|