/******************************************************************************
 * Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *****************************************************************************/
/*! \file
    \brief Architecture-specific operators on memory added for GFX9
*/
// reference:
//   https://github.com/llvm/llvm-project/blob/main/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll

#ifndef INTRINSIC_MEM_ACCESS_H
#define INTRINSIC_MEM_ACCESS_H

#if defined(__clang__) && defined(__HIP__)

#if(defined(__NVCC__) || defined(__HIPCC__)) \
    || (defined(__clang__) && (defined(__CUDA__)) || defined(__HIP__))
#define INLINEDEVICE __forceinline__ __device__
#elif defined(__CUDACC_RTC__)
#define INLINEDEVICE __forceinline__ __device__
#else
#define INLINEDEVICE inline
#endif

#if defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || defined(__gfx908__) \
    || defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) // test device
#define USE_GFX_BUFFER_INTRINSIC
#define BUFFER_RESOURCE_3RD_DWORD 0x00020000
#elif defined(__gfx1030__) // special device
#define USE_GFX_BUFFER_INTRINSIC
#define BUFFER_RESOURCE_3RD_DWORD 0x31014000
#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) \
    || defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
#define USE_GFX_BUFFER_INTRINSIC
#define BUFFER_RESOURCE_3RD_DWORD 0x31004000
#elif defined(__gfx1200__) || defined(__gfx1201__)
#define USE_GFX_BUFFER_INTRINSIC
#define BUFFER_RESOURCE_3RD_DWORD 0x30020000
#elif defined(__gfx1250__)
#define USE_GFX_BUFFER_INTRINSIC
#define BUFFER_RESOURCE_3RD_DWORD 0x00000000
#else // not support
#define BUFFER_RESOURCE_3RD_DWORD -1
#endif

/// Controls AMD gfx arch cache operations
struct CacheOperation
{
    enum Kind
    {
        /// Cache at all levels - accessed again
        Always,
        /// Cache at global level; glc = 1
        Global,
        /// Streaming - likely to be accessed once; slc = 1
        Streaming,
        /// Indicates the line will not be used again, glc = 1; slc = 1
        LastUse
    };
};

using float16_t = _Float16;
using float32_t = float;

template <typename T, int N>
struct NativeVector
{
    using type = T __attribute__((ext_vector_type(N)));
};

using float32x2_t = NativeVector<float, 2>::type;
using float32x4_t = NativeVector<float, 4>::type;

using int32x4_t = NativeVector<int, 4>::type;
using int32x2_t = NativeVector<int, 2>::type;

////////////////////////////////////////////////////////////////////////////////////////////////////

struct alignas(16) BufferResource
{
    union Desc
    {
        int32x4_t d128;
        void*     d64[2];
        uint32_t  d32[4];
    };

    INLINEDEVICE
    BufferResource(void const* base_addr, uint64_t num_records = (0xFFFFFFFF - 1))
    {
        //   GFX1250:
        //      base address bits [56:0] (57 bits)
        //      num_records bits [101:57] (45 bits)
        //   Other archs:
        //      base address bits [47:0] (47 bits)
        //      num_records bits [95:64] (32 bits)

#if defined(__gfx1250__)
        uint64_t addr = reinterpret_cast<uint64_t>(const_cast<void*>(base_addr));

        // bits [56:0] - base address (57 bits)
        desc_.d32[0] = static_cast<uint32_t>(addr);
        desc_.d32[1] = static_cast<uint32_t>(addr >> 32) & 0x01FFFFFFu;  // bits 32-56

        // bits [101:57] - num_records (45 bits)
        desc_.d32[1] |= static_cast<uint32_t>((num_records & 0x7Fu) << 25);   // bits 57-63
        desc_.d32[2] = static_cast<uint32_t>((num_records >> 7) & 0xFFFFFFFFu); // bits 64-95
        desc_.d32[3] = (BUFFER_RESOURCE_3RD_DWORD & 0xFFFFFFC0u) |
                       static_cast<uint32_t>((num_records >> 39) & 0x3Fu);     // bits 96-101
#else
        desc_.d64[0] = const_cast<void*>(base_addr);
        desc_.d32[2] = static_cast<uint32_t>(num_records);
        desc_.d32[3] = BUFFER_RESOURCE_3RD_DWORD;
#endif
    }

    INLINEDEVICE
    operator int32x4_t()
    {
        // return desc_.d128; // NOTE HIP: Crashes compiler; see below

        /// This hack is to enforce scalarization of the variable "base_addr", where in some
        /// circumstances it becomes vectorized and then in turn causes illegal lowering to GCN ISA
        /// since compiler effectively tries to stuff VGPRs in slots where it only accepts SGPRs
        Desc ret;
        ret.d32[0] = __builtin_amdgcn_readfirstlane(desc_.d32[0]);
        ret.d32[1] = __builtin_amdgcn_readfirstlane(desc_.d32[1]);
        ret.d64[1] = desc_.d64[1];
        return ret.d128;
        ///
    }

    Desc desc_;
};

INLINEDEVICE
void const* splitBufferOffset(void const* base_ptr, uint64_t voffset, uint32_t& voffset_lo)
{
    voffset_lo     = static_cast<uint32_t>(voffset);
    uint64_t base  = reinterpret_cast<uint64_t>(const_cast<void*>(base_ptr));
    base          += voffset & 0xFFFFFFFF00000000ull;
    return reinterpret_cast<void const*>(base);
}

////////////////////////////////////////////////////////////////////////////////////////////////////

///
/// Load
///

// 1 byte
__device__ char
    llvm_amdgcn_raw_buffer_load_i8(int32x4_t buffer_resource,
                                   uint32_t  voffset,
                                   uint32_t  soffset,
                                   int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.load.i8");

// 2 bytes
__device__ float16_t
    llvm_amdgcn_raw_buffer_load_f16(int32x4_t buffer_resource,
                                    uint32_t  voffset,
                                    uint32_t  soffset,
                                    int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.load.f16");

// 4 bytes
__device__ float32_t
    llvm_amdgcn_raw_buffer_load_f32(int32x4_t buffer_resource,
                                    uint32_t  voffset,
                                    uint32_t  soffset,
                                    int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.load.f32");

// 8 bytes
__device__ float32x2_t
    llvm_amdgcn_raw_buffer_load_f32x2(int32x4_t buffer_resource,
                                      uint32_t  voffset,
                                      uint32_t  soffset,
                                      int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v2f32");

// 16 bytes
__device__ float32x4_t
    llvm_amdgcn_raw_buffer_load_f32x4(int32x4_t buffer_resource,
                                      uint32_t  voffset,
                                      uint32_t  soffset,
                                      int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v4f32");

// 4 bytes
__device__ int32_t
    llvm_amdgcn_s_buffer_load_i32(int32x4_t buffer_resource,
                                  uint32_t  soffset,
                                  int32_t   cache_op) __asm("llvm.amdgcn.s.buffer.load.i32");

// 8 bytes
__device__ int32x2_t
    llvm_amdgcn_s_buffer_load_i32x2(int32x4_t buffer_resource,
                                    uint32_t  soffset,
                                    int32_t   cache_op) __asm("llvm.amdgcn.s.buffer.load.v2i32");

// 16 bytes
__device__ int32x4_t
    llvm_amdgcn_s_buffer_load_i32x4(int32x4_t buffer_resource,
                                    uint32_t  soffset,
                                    int32_t   cache_op) __asm("llvm.amdgcn.s.buffer.load.v4i32");

///
/// Store
///

// 1 byte
__device__ void
    llvm_amdgcn_raw_buffer_store_i8(char      data,
                                    int32x4_t buffer_resource,
                                    uint32_t  voffset,
                                    uint32_t  soffset,
                                    int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.store.i8");

// 2 bytes
__device__ void
    llvm_amdgcn_raw_buffer_store_f16(float16_t data,
                                     int32x4_t buffer_resource,
                                     uint32_t  voffset,
                                     uint32_t  soffset,
                                     int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.store.f16");

// 4 bytes
__device__ void
    llvm_amdgcn_raw_buffer_store_f32(float32_t data,
                                     int32x4_t buffer_resource,
                                     uint32_t  voffset,
                                     uint32_t  soffset,
                                     int32_t   cache_op) __asm("llvm.amdgcn.raw.buffer.store.f32");

// 8 bytes
__device__ void llvm_amdgcn_raw_buffer_store_f32x2(
    float32x2_t data,
    int32x4_t   buffer_resource,
    uint32_t    voffset,
    uint32_t    soffset,
    int32_t     cache_op) __asm("llvm.amdgcn.raw.buffer.store.v2f32");

// 16 bytes
__device__ void llvm_amdgcn_raw_buffer_store_f32x4(
    float32x4_t data,
    int32x4_t   buffer_resource,
    uint32_t    voffset,
    uint32_t    soffset,
    int32_t     cache_op) __asm("llvm.amdgcn.raw.buffer.store.v4f32");

////////////////////////////////////////////////////////////////////////////////////////////////////

template <
    /// Fragment type to store loaded data
    typename AccessType,
    /// The bytes of loading
    int LoadBytes,
    /// Cache operation
    CacheOperation::Kind cache_op = CacheOperation::Always>
struct buffer_load;

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_load<AccessType, 1, cache_op>
{
    INLINEDEVICE
    buffer_load() {}

    INLINEDEVICE
    buffer_load(AccessType& D,
                void const* base_ptr,
                uint32_t    voffset,
                uint32_t    soffset,
                uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        char           ret = llvm_amdgcn_raw_buffer_load_i8(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr,
                    uint32_t    voffset,
                    uint32_t    soffset,
                    uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        char           ret = llvm_amdgcn_raw_buffer_load_i8(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_load<AccessType, 2, cache_op>
{
    INLINEDEVICE
    buffer_load() {}

    INLINEDEVICE
    buffer_load(AccessType& D,
                void const* base_ptr,
                uint32_t    voffset,
                uint32_t    soffset,
                uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float16_t      ret = llvm_amdgcn_raw_buffer_load_f16(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr,
                    uint32_t    voffset,
                    uint32_t    soffset,
                    uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float16_t      ret = llvm_amdgcn_raw_buffer_load_f16(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_load<AccessType, 4, cache_op>
{
    INLINEDEVICE
    buffer_load() {}

    INLINEDEVICE
    buffer_load(AccessType& D,
                void const* base_ptr,
                uint32_t    voffset,
                uint32_t    soffset,
                uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32_t      ret = llvm_amdgcn_raw_buffer_load_f32(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr,
                    uint32_t    voffset,
                    uint32_t    soffset,
                    uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32_t      ret = llvm_amdgcn_raw_buffer_load_f32(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_load<AccessType, 8, cache_op>
{
    INLINEDEVICE
    buffer_load() {}

    INLINEDEVICE
    buffer_load(AccessType& D,
                void const* base_ptr,
                uint32_t    voffset,
                uint32_t    soffset,
                uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x2_t    ret = llvm_amdgcn_raw_buffer_load_f32x2(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr,
                    uint32_t    voffset,
                    uint32_t    soffset,
                    uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x2_t    ret = llvm_amdgcn_raw_buffer_load_f32x2(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_load<AccessType, 16, cache_op>
{
    INLINEDEVICE
    buffer_load() {}

    INLINEDEVICE
    buffer_load(AccessType& D,
                void const* base_ptr,
                uint32_t    voffset,
                uint32_t    soffset,
                uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x4_t    ret = llvm_amdgcn_raw_buffer_load_f32x4(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr,
                    uint32_t    voffset,
                    uint32_t    soffset,
                    uint32_t    num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x4_t    ret = llvm_amdgcn_raw_buffer_load_f32x4(
            buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <
    /// Fragment type to store loaded data
    typename AccessType,
    /// The bytes of loading
    int LoadBytes,
    /// Cache operation
    CacheOperation::Kind cache_op = CacheOperation::Always>
struct s_buffer_load;

template <typename AccessType, CacheOperation::Kind cache_op>
struct s_buffer_load<AccessType, 4, cache_op>
{
    INLINEDEVICE
    s_buffer_load() {}

    INLINEDEVICE
    s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32_t        ret = llvm_amdgcn_s_buffer_load_i32(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32_t        ret = llvm_amdgcn_s_buffer_load_i32(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct s_buffer_load<AccessType, 8, cache_op>
{
    INLINEDEVICE
    s_buffer_load() {}

    INLINEDEVICE
    s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32x2_t      ret = llvm_amdgcn_s_buffer_load_i32x2(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32x2_t      ret = llvm_amdgcn_s_buffer_load_i32x2(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct s_buffer_load<AccessType, 16, cache_op>
{
    INLINEDEVICE
    s_buffer_load() {}

    INLINEDEVICE
    s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32x4_t      ret = llvm_amdgcn_s_buffer_load_i32x4(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        D = *reinterpret_cast<AccessType*>(&ret);
    }

    INLINEDEVICE
    AccessType load(void const* base_ptr, uint32_t soffset)
    {
        BufferResource buffer_rsc(base_ptr);
        int32x4_t      ret = llvm_amdgcn_s_buffer_load_i32x4(
            buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op);
        return *reinterpret_cast<AccessType*>(&ret);
    }
};

////////////////////////////////////////////////////////////////////////////////////////////////////

template <
    /// Fragment type to store loaded data
    typename AccessType,
    /// The width of loading
    int NumElements,
    /// Cache operation
    CacheOperation::Kind cache_op = CacheOperation::Always>
struct buffer_store;

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_store<AccessType, 1, cache_op>
{
    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint32_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        char           data = *reinterpret_cast<char const*>(&D);
        llvm_amdgcn_raw_buffer_store_i8(
            data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }

    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint64_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        uint32_t voffset_lo = 0;
        base_ptr            = splitBufferOffset(base_ptr, voffset, voffset_lo);
        BufferResource buffer_rsc(base_ptr, num_records);
        char           data = *reinterpret_cast<char const*>(&D);
        llvm_amdgcn_raw_buffer_store_i8(
            data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_store<AccessType, 2, cache_op>
{
    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint32_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float16_t      data = *reinterpret_cast<float16_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f16(
            data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }

    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint64_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        uint32_t voffset_lo = 0;
        base_ptr            = splitBufferOffset(base_ptr, voffset, voffset_lo);
        BufferResource buffer_rsc(base_ptr, num_records);
        float16_t      data = *reinterpret_cast<float16_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f16(
            data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_store<AccessType, 4, cache_op>
{
    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint32_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32_t      data = *reinterpret_cast<float32_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32(
            data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }

    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint64_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        uint32_t voffset_lo = 0;
        base_ptr            = splitBufferOffset(base_ptr, voffset, voffset_lo);
        BufferResource buffer_rsc(base_ptr, num_records);
        float32_t      data = *reinterpret_cast<float32_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32(
            data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_store<AccessType, 8, cache_op>
{
    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint32_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x2_t    data = *reinterpret_cast<float32x2_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32x2(
            data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }

    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint64_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        uint32_t voffset_lo = 0;
        base_ptr            = splitBufferOffset(base_ptr, voffset, voffset_lo);
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x2_t    data = *reinterpret_cast<float32x2_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32x2(
            data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }
};

template <typename AccessType, CacheOperation::Kind cache_op>
struct buffer_store<AccessType, 16, cache_op>
{
    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint32_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x4_t    data = *reinterpret_cast<float32x4_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32x4(
            data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }

    INLINEDEVICE
    buffer_store(const AccessType& D,
                 void const*       base_ptr,
                 uint64_t          voffset,
                 uint32_t          soffset,
                 uint32_t          num_records = (0xFFFFFFFF - 1))
    {
        uint32_t voffset_lo = 0;
        base_ptr            = splitBufferOffset(base_ptr, voffset, voffset_lo);
        BufferResource buffer_rsc(base_ptr, num_records);
        float32x4_t    data = *reinterpret_cast<float32x4_t const*>(&D);
        llvm_amdgcn_raw_buffer_store_f32x4(
            data, buffer_rsc, voffset_lo, __builtin_amdgcn_readfirstlane(soffset), cache_op);
    }
};

////////////////////////////////////////////////////////////////////////////////////////////////////

#endif // defined(__clang__) && defined(__HIP__)

#endif // INTRINSIC_MEM_ACCESS_H