derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
68 lines
2.9 KiB
C++
68 lines
2.9 KiB
C++
// -*- C++ -*-
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of libcu++, the C++ Standard Library for your entire system,
|
|
// under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef _CUDA_PTX_BARRIER_EXPECT_TX_H_
|
|
#define _CUDA_PTX_BARRIER_EXPECT_TX_H_
|
|
|
|
#include <cuda/std/detail/__config>
|
|
|
|
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
|
# pragma GCC system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
|
# pragma clang system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
|
# pragma system_header
|
|
#endif // no system header
|
|
|
|
#if _CCCL_HAS_CUDA_COMPILER()
|
|
# if __cccl_ptx_isa >= 800
|
|
|
|
# include <cuda/__barrier/barrier_block_scope.h>
|
|
# include <cuda/__barrier/barrier_native_handle.h>
|
|
# include <cuda/__ptx/ptx_dot_variants.h>
|
|
# include <cuda/__ptx/ptx_helper_functions.h>
|
|
# include <cuda/std/__atomic/scopes.h>
|
|
# include <cuda/std/cstdint>
|
|
|
|
_LIBCUDACXX_BEGIN_NAMESPACE_CUDA_DEVICE
|
|
|
|
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_expect_tx_is_not_supported_before_SM_90__();
|
|
_CCCL_DEVICE inline void
|
|
barrier_expect_tx(barrier<thread_scope_block>& __b, _CUDA_VSTD::ptrdiff_t __transaction_count_update)
|
|
{
|
|
_CCCL_ASSERT(__isShared(barrier_native_handle(__b)), "Barrier must be located in local shared memory.");
|
|
_CCCL_ASSERT(__transaction_count_update >= 0, "Transaction count update must be non-negative.");
|
|
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#contents-of-the-mbarrier-object
|
|
_CCCL_ASSERT(__transaction_count_update <= (1 << 20) - 1, "Transaction count update cannot exceed 2^20 - 1.");
|
|
|
|
// We do not check for the statespace of the barrier here. This is
|
|
// on purpose. This allows debugging tools like memcheck/racecheck
|
|
// to detect that we are passing a pointer with the wrong state
|
|
// space to mbarrier.arrive. If we checked for the state space here,
|
|
// and __trap() if wrong, then those tools would not be able to help
|
|
// us in release builds. In debug builds, the error would be caught
|
|
// by the asserts at the top of this function.
|
|
// On architectures pre-sm90, arrive_tx is not supported.
|
|
NV_IF_ELSE_TARGET(
|
|
NV_PROVIDES_SM_90,
|
|
(auto __bh = __cvta_generic_to_shared(barrier_native_handle(__b));
|
|
asm("mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" : : "r"(static_cast<_CUDA_VSTD::uint32_t>(__bh)),
|
|
"r"(static_cast<_CUDA_VSTD::uint32_t>(__transaction_count_update)) : "memory");),
|
|
(__cuda_ptx_barrier_expect_tx_is_not_supported_before_SM_90__();));
|
|
}
|
|
|
|
_LIBCUDACXX_END_NAMESPACE_CUDA_DEVICE
|
|
|
|
# endif // __cccl_ptx_isa >= 800
|
|
#endif // _CCCL_CUDA_COMPILER
|
|
|
|
#endif // _CUDA_PTX_BARRIER_EXPECT_TX_H_
|