derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
696 lines
25 KiB
C++
696 lines
25 KiB
C++
/******************************************************************************
|
|
* Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
*****************************************************************************/
|
|
/*! \file
|
|
\brief Architecture-specific operators on memory added for GFX9
|
|
*/
|
|
// reference:
|
|
// https://github.com/llvm/llvm-project/blob/main/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
|
|
|
|
#ifndef INTRINSIC_MEM_ACCESS_H
|
|
#define INTRINSIC_MEM_ACCESS_H
|
|
|
|
#if defined(__clang__) && defined(__HIP__)
|
|
|
|
#if(defined(__NVCC__) || defined(__HIPCC__)) \
|
|
|| (defined(__clang__) && (defined(__CUDA__)) || defined(__HIP__))
|
|
#define INLINEDEVICE __forceinline__ __device__
|
|
#elif defined(__CUDACC_RTC__)
|
|
#define INLINEDEVICE __forceinline__ __device__
|
|
#else
|
|
#define INLINEDEVICE inline
|
|
#endif
|
|
|
|
#if defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || defined(__gfx908__) \
|
|
|| defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) // test device
|
|
#define USE_GFX_BUFFER_INTRINSIC
|
|
#define BUFFER_RESOURCE_3RD_DWORD 0x00020000
|
|
#elif defined(__gfx1030__) // special device
|
|
#define USE_GFX_BUFFER_INTRINSIC
|
|
#define BUFFER_RESOURCE_3RD_DWORD 0x31014000
|
|
#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) \
|
|
|| defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
|
|
#define USE_GFX_BUFFER_INTRINSIC
|
|
#define BUFFER_RESOURCE_3RD_DWORD 0x31004000
|
|
#elif defined(__gfx1200__) || defined(__gfx1201__)
|
|
#define USE_GFX_BUFFER_INTRINSIC
|
|
#define BUFFER_RESOURCE_3RD_DWORD 0x30020000
|
|
#elif defined(__gfx1250__)
|
|
#define USE_GFX_BUFFER_INTRINSIC
|
|
#define BUFFER_RESOURCE_3RD_DWORD 0x00000000
|
|
#else // not support
|
|
#define BUFFER_RESOURCE_3RD_DWORD -1
|
|
#endif
|
|
|
|
/// Controls AMD gfx arch cache operations
|
|
struct CacheOperation
|
|
{
|
|
enum Kind
|
|
{
|
|
/// Cache at all levels - accessed again
|
|
Always,
|
|
/// Cache at global level; glc = 1
|
|
Global,
|
|
/// Streaming - likely to be accessed once; slc = 1
|
|
Streaming,
|
|
/// Indicates the line will not be used again, glc = 1; slc = 1
|
|
LastUse
|
|
};
|
|
};
|
|
|
|
using float16_t = _Float16;
|
|
using float32_t = float;
|
|
|
|
template <typename T, int N>
|
|
struct NativeVector
|
|
{
|
|
using type = T __attribute__((ext_vector_type(N)));
|
|
};
|
|
|
|
using float32x2_t = NativeVector<float, 2>::type;
|
|
using float32x4_t = NativeVector<float, 4>::type;
|
|
|
|
using int32x4_t = NativeVector<int, 4>::type;
|
|
using int32x2_t = NativeVector<int, 2>::type;
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
struct alignas(16) BufferResource
|
|
{
|
|
union Desc
|
|
{
|
|
int32x4_t d128;
|
|
void* d64[2];
|
|
uint32_t d32[4];
|
|
};
|
|
|
|
INLINEDEVICE
|
|
BufferResource(void const* base_addr, uint64_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
// GFX1250:
|
|
// base address bits [56:0] (57 bits)
|
|
// num_records bits [101:57] (45 bits)
|
|
// Other archs:
|
|
// base address bits [47:0] (47 bits)
|
|
// num_records bits [95:64] (32 bits)
|
|
|
|
#if defined(__gfx1250__)
|
|
uint64_t addr = reinterpret_cast<uint64_t>(const_cast<void*>(base_addr));
|
|
|
|
// bits [56:0] - base address (57 bits)
|
|
desc_.d32[0] = static_cast<uint32_t>(addr);
|
|
desc_.d32[1] = static_cast<uint32_t>(addr >> 32) & 0x01FFFFFFu; // bits 32-56
|
|
|
|
// bits [101:57] - num_records (45 bits)
|
|
desc_.d32[1] |= static_cast<uint32_t>((num_records & 0x7Fu) << 25); // bits 57-63
|
|
desc_.d32[2] = static_cast<uint32_t>((num_records >> 7) & 0xFFFFFFFFu); // bits 64-95
|
|
desc_.d32[3] = (BUFFER_RESOURCE_3RD_DWORD & 0xFFFFFFC0u) |
|
|
static_cast<uint32_t>((num_records >> 39) & 0x3Fu); // bits 96-101
|
|
#else
|
|
desc_.d64[0] = const_cast<void*>(base_addr);
|
|
desc_.d32[2] = static_cast<uint32_t>(num_records);
|
|
desc_.d32[3] = BUFFER_RESOURCE_3RD_DWORD;
|
|
#endif
|
|
}
|
|
|
|
INLINEDEVICE
|
|
operator int32x4_t()
|
|
{
|
|
// return desc_.d128; // NOTE HIP: Crashes compiler; see below
|
|
|
|
/// This hack is to enforce scalarization of the variable "base_addr", where in some
|
|
/// circumstances it becomes vectorized and then in turn causes illegal lowering to GCN ISA
|
|
/// since compiler effectively tries to stuff VGPRs in slots where it only accepts SGPRs
|
|
Desc ret;
|
|
ret.d32[0] = __builtin_amdgcn_readfirstlane(desc_.d32[0]);
|
|
ret.d32[1] = __builtin_amdgcn_readfirstlane(desc_.d32[1]);
|
|
ret.d64[1] = desc_.d64[1];
|
|
return ret.d128;
|
|
///
|
|
}
|
|
|
|
Desc desc_;
|
|
};
|
|
|
|
INLINEDEVICE
|
|
void const* splitBufferOffset(void const* base_ptr, uint64_t voffset, uint32_t& voffset_lo)
|
|
{
|
|
voffset_lo = static_cast<uint32_t>(voffset);
|
|
uint64_t base = reinterpret_cast<uint64_t>(const_cast<void*>(base_ptr));
|
|
base += voffset & 0xFFFFFFFF00000000ull;
|
|
return reinterpret_cast<void const*>(base);
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
///
|
|
/// Load
|
|
///
|
|
|
|
// 1 byte
|
|
__device__ char
|
|
llvm_amdgcn_raw_buffer_load_i8(int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.i8");
|
|
|
|
// 2 bytes
|
|
__device__ float16_t
|
|
llvm_amdgcn_raw_buffer_load_f16(int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f16");
|
|
|
|
// 4 bytes
|
|
__device__ float32_t
|
|
llvm_amdgcn_raw_buffer_load_f32(int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f32");
|
|
|
|
// 8 bytes
|
|
__device__ float32x2_t
|
|
llvm_amdgcn_raw_buffer_load_f32x2(int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v2f32");
|
|
|
|
// 16 bytes
|
|
__device__ float32x4_t
|
|
llvm_amdgcn_raw_buffer_load_f32x4(int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v4f32");
|
|
|
|
// 4 bytes
|
|
__device__ int32_t
|
|
llvm_amdgcn_s_buffer_load_i32(int32x4_t buffer_resource,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.i32");
|
|
|
|
// 8 bytes
|
|
__device__ int32x2_t
|
|
llvm_amdgcn_s_buffer_load_i32x2(int32x4_t buffer_resource,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v2i32");
|
|
|
|
// 16 bytes
|
|
__device__ int32x4_t
|
|
llvm_amdgcn_s_buffer_load_i32x4(int32x4_t buffer_resource,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v4i32");
|
|
|
|
///
|
|
/// Store
|
|
///
|
|
|
|
// 1 byte
|
|
__device__ void
|
|
llvm_amdgcn_raw_buffer_store_i8(char data,
|
|
int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.i8");
|
|
|
|
// 2 bytes
|
|
__device__ void
|
|
llvm_amdgcn_raw_buffer_store_f16(float16_t data,
|
|
int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f16");
|
|
|
|
// 4 bytes
|
|
__device__ void
|
|
llvm_amdgcn_raw_buffer_store_f32(float32_t data,
|
|
int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f32");
|
|
|
|
// 8 bytes
|
|
__device__ void llvm_amdgcn_raw_buffer_store_f32x2(
|
|
float32x2_t data,
|
|
int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v2f32");
|
|
|
|
// 16 bytes
|
|
__device__ void llvm_amdgcn_raw_buffer_store_f32x4(
|
|
float32x4_t data,
|
|
int32x4_t buffer_resource,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v4f32");
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
template <
|
|
/// Fragment type to store loaded data
|
|
typename AccessType,
|
|
/// The bytes of loading
|
|
int LoadBytes,
|
|
/// Cache operation
|
|
CacheOperation::Kind cache_op = CacheOperation::Always>
|
|
struct buffer_load;
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_load<AccessType, 1, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
buffer_load(AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
char ret = llvm_amdgcn_raw_buffer_load_i8(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
char ret = llvm_amdgcn_raw_buffer_load_i8(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_load<AccessType, 2, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
buffer_load(AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float16_t ret = llvm_amdgcn_raw_buffer_load_f16(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float16_t ret = llvm_amdgcn_raw_buffer_load_f16(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_load<AccessType, 4, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
buffer_load(AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32_t ret = llvm_amdgcn_raw_buffer_load_f32(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32_t ret = llvm_amdgcn_raw_buffer_load_f32(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_load<AccessType, 8, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
buffer_load(AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_load<AccessType, 16, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
buffer_load(AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4(
|
|
buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <
|
|
/// Fragment type to store loaded data
|
|
typename AccessType,
|
|
/// The bytes of loading
|
|
int LoadBytes,
|
|
/// Cache operation
|
|
CacheOperation::Kind cache_op = CacheOperation::Always>
|
|
struct s_buffer_load;
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct s_buffer_load<AccessType, 4, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
s_buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32_t ret = llvm_amdgcn_s_buffer_load_i32(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32_t ret = llvm_amdgcn_s_buffer_load_i32(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct s_buffer_load<AccessType, 8, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
s_buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct s_buffer_load<AccessType, 16, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
s_buffer_load() {}
|
|
|
|
INLINEDEVICE
|
|
s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
D = *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
AccessType load(void const* base_ptr, uint32_t soffset)
|
|
{
|
|
BufferResource buffer_rsc(base_ptr);
|
|
int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4(
|
|
buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
return *reinterpret_cast<AccessType*>(&ret);
|
|
}
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
template <
|
|
/// Fragment type to store loaded data
|
|
typename AccessType,
|
|
/// The width of loading
|
|
int NumElements,
|
|
/// Cache operation
|
|
CacheOperation::Kind cache_op = CacheOperation::Always>
|
|
struct buffer_store;
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_store<AccessType, 1, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
char data = *reinterpret_cast<char const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_i8(
|
|
data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint64_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
uint32_t voffset_lo = 0;
|
|
base_ptr = splitBufferOffset(base_ptr, voffset, voffset_lo);
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
char data = *reinterpret_cast<char const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_i8(
|
|
data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_store<AccessType, 2, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float16_t data = *reinterpret_cast<float16_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f16(
|
|
data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint64_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
uint32_t voffset_lo = 0;
|
|
base_ptr = splitBufferOffset(base_ptr, voffset, voffset_lo);
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float16_t data = *reinterpret_cast<float16_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f16(
|
|
data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_store<AccessType, 4, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32_t data = *reinterpret_cast<float32_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32(
|
|
data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint64_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
uint32_t voffset_lo = 0;
|
|
base_ptr = splitBufferOffset(base_ptr, voffset, voffset_lo);
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32_t data = *reinterpret_cast<float32_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32(
|
|
data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_store<AccessType, 8, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x2_t data = *reinterpret_cast<float32x2_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32x2(
|
|
data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint64_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
uint32_t voffset_lo = 0;
|
|
base_ptr = splitBufferOffset(base_ptr, voffset, voffset_lo);
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x2_t data = *reinterpret_cast<float32x2_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32x2(
|
|
data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
};
|
|
|
|
template <typename AccessType, CacheOperation::Kind cache_op>
|
|
struct buffer_store<AccessType, 16, cache_op>
|
|
{
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint32_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x4_t data = *reinterpret_cast<float32x4_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32x4(
|
|
data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
|
|
INLINEDEVICE
|
|
buffer_store(const AccessType& D,
|
|
void const* base_ptr,
|
|
uint64_t voffset,
|
|
uint32_t soffset,
|
|
uint32_t num_records = (0xFFFFFFFF - 1))
|
|
{
|
|
uint32_t voffset_lo = 0;
|
|
base_ptr = splitBufferOffset(base_ptr, voffset, voffset_lo);
|
|
BufferResource buffer_rsc(base_ptr, num_records);
|
|
float32x4_t data = *reinterpret_cast<float32x4_t const*>(&D);
|
|
llvm_amdgcn_raw_buffer_store_f32x4(
|
|
data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
|
|
}
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
#endif // defined(__clang__) && defined(__HIP__)
|
|
|
|
#endif // INTRINSIC_MEM_ACCESS_H
|