; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefix=GFX942

; =============================================================================
; Flat atomic fadd - f32
; =============================================================================

define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat(ptr %ptr) {
; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v2, 4.0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT:    buffer_wbl2 sc0 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    flat_atomic_add_f32 v[0:1], v2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_inv sc0 sc1
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat_ieee(ptr %ptr) #0 {
; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v2, 4.0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT:    buffer_wbl2 sc0 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    flat_atomic_add_f32 v[0:1], v2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_inv sc0 sc1
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define float @flat_atomic_fadd_f32_rtn_pat(ptr %ptr, float %data) {
; GFX942-LABEL: flat_atomic_fadd_f32_rtn_pat:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    v_mov_b32_e32 v2, 4.0
; GFX942-NEXT:    buffer_wbl2 sc0 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    flat_atomic_add_f32 v0, v[0:1], v2 sc0 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_inv sc0 sc1
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
  ret float %ret
}

; =============================================================================
; Flat atomic fadd - v2f16
; =============================================================================

define <2 x half> @flat_atomic_fadd_ret_v2f16_agent_offset(ptr %ptr, <2 x half> %val) {
; GFX942-LABEL: flat_atomic_fadd_ret_v2f16_agent_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_wbl2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:1024 sc0
; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_inv sc1
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
  %result = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
  ret <2 x half> %result
}

define void @flat_atomic_fadd_noret_v2f16_agent_offset(ptr %ptr, <2 x half> %val) {
; GFX942-LABEL: flat_atomic_fadd_noret_v2f16_agent_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_wbl2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    flat_atomic_pk_add_f16 v[0:1], v2 offset:1024
; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_inv sc1
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
  %unused = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
  ret void
}

; =============================================================================
; Global atomic fadd - f32
; =============================================================================

define amdgpu_ps void @global_atomic_fadd_f32_no_rtn_atomicrmw(ptr addrspace(1) %ptr, float %data) {
; GFX942-LABEL: global_atomic_fadd_f32_no_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    global_atomic_add_f32 v[0:1], v2, off
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_ps float @global_atomic_fadd_f32_rtn_atomicrmw(ptr addrspace(1) %ptr, float %data) {
; GFX942-LABEL: global_atomic_fadd_f32_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    global_atomic_add_f32 v0, v[0:1], v2, off sc0
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    ; return to shader part epilog
  %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret float %ret
}

; =============================================================================
; Global atomic fadd - v2f16
; =============================================================================

define <2 x half> @global_atomic_fadd_ret_v2f16_agent_offset(ptr addrspace(1) %ptr, <2 x half> %val) {
; GFX942-LABEL: global_atomic_fadd_ret_v2f16_agent_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_wbl2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    global_atomic_pk_add_f16 v0, v[0:1], v2, off offset:1024 sc0
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    buffer_inv sc1
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i32 256
  %result = atomicrmw fadd ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
  ret <2 x half> %result
}

define void @global_atomic_fadd_noret_v2f16_agent_offset(ptr addrspace(1) %ptr, <2 x half> %val) {
; GFX942-LABEL: global_atomic_fadd_noret_v2f16_agent_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    buffer_wbl2 sc1
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    global_atomic_pk_add_f16 v[0:1], v2, off offset:1024
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    buffer_inv sc1
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i32 256
  %unused = atomicrmw fadd ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
  ret void
}

; =============================================================================
; Global atomic fadd - f64
; =============================================================================

define amdgpu_ps void @global_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(1) %ptr, double %data) {
; GFX942-LABEL: global_atomic_fadd_f64_no_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %ptr, double %data) {
; GFX942-LABEL: global_atomic_fadd_f64_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    v_readfirstlane_b32 s0, v0
; GFX942-NEXT:    v_readfirstlane_b32 s1, v1
; GFX942-NEXT:    ; return to shader part epilog
  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret double %ret
}

; =============================================================================
; Local atomic fadd - f32
; =============================================================================

define amdgpu_ps void @local_atomic_fadd_f32_no_rtn_atomicrmw(ptr addrspace(3) %ptr, float %data) {
; GFX942-LABEL: local_atomic_fadd_f32_no_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    ds_add_f32 v0, v1
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr addrspace(3) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_ps float @local_atomic_fadd_f32_rtn_atomicrmw(ptr addrspace(3) %ptr, float %data) {
; GFX942-LABEL: local_atomic_fadd_f32_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    ds_add_rtn_f32 v0, v0, v1
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    ; return to shader part epilog
  %ret = atomicrmw fadd ptr addrspace(3) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret float %ret
}

; =============================================================================
; Local atomic fadd - v2f16
; =============================================================================

define <2 x half> @local_atomic_fadd_ret_v2f16_offset(ptr addrspace(3) %ptr, <2 x half> %val) {
; GFX942-LABEL: local_atomic_fadd_ret_v2f16_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    ds_pk_add_rtn_f16 v0, v0, v1 offset:65532
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
  %result = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst
  ret <2 x half> %result
}

define void @local_atomic_fadd_noret_v2f16_offset(ptr addrspace(3) %ptr, <2 x half> %val) {
; GFX942-LABEL: local_atomic_fadd_noret_v2f16_offset:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    ds_pk_add_f16 v0, v1 offset:65532
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
  %unused = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst
  ret void
}

; =============================================================================
; Local atomic fadd - f64
; =============================================================================

define amdgpu_ps void @local_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(3) %ptr, double %data) {
; GFX942-LABEL: local_atomic_fadd_f64_no_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    v_mov_b32_e32 v4, v1
; GFX942-NEXT:    v_mov_b32_e32 v5, v2
; GFX942-NEXT:    ds_add_f64 v0, v[4:5]
; GFX942-NEXT:    s_endpgm
  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_ps double @local_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(3) %ptr, double %data) {
; GFX942-LABEL: local_atomic_fadd_f64_rtn_atomicrmw:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    v_mov_b32_e32 v4, v1
; GFX942-NEXT:    v_mov_b32_e32 v5, v2
; GFX942-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[4:5]
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_readfirstlane_b32 s0, v0
; GFX942-NEXT:    v_readfirstlane_b32 s1, v1
; GFX942-NEXT:    ; return to shader part epilog
  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
  ret double %ret
}

attributes #0 = { denormal_fpenv(float: ieee|ieee) }

!0 = !{}