; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12PLUS %s
; RUN: llc -mtriple=amdgcn -global-isel=1 -global-isel-abort=2 -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12PLUS %s
; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx1310 < %s | FileCheck -check-prefixes=GFX12PLUS %s
; RUN: llc -mtriple=amdgcn -global-isel=1 -global-isel-abort=2 -mcpu=gfx1310 < %s | FileCheck -check-prefixes=GFX12PLUS %s

define amdgpu_ps float @atomic_pk_add_f16_1d_v2(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v2:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
; GFX12PLUS-NEXT:    s_wait_loadcnt 0x0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %out = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  %out_float = bitcast <2 x half> %out to float
  ret float %out_float
}

define amdgpu_ps float @atomic_pk_add_f16_1d_v2_nt(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v2_nt:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN
; GFX12PLUS-NEXT:    s_wait_loadcnt 0x0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %out = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
  %out_float = bitcast <2 x half> %out to float
  ret float %out_float
}

define amdgpu_ps float @atomic_pk_add_f16_1d_v2_noret(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v2_noret:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %unused = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_f16_1d_v4(<8 x i32> inreg %rsrc, <4 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v4:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
; GFX12PLUS-NEXT:    s_wait_loadcnt 0x0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %out = call <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  %out_v2f32 = bitcast <4 x half> %out to <2 x float>
  %out0 = extractelement <2 x float> %out_v2f32, i32 0
  ret float %out0
}

define amdgpu_ps float @atomic_pk_add_f16_1d_v4_noret(<8 x i32> inreg %rsrc, <4 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v4_noret:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %unused = call <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_bf16_1d_v2(<8 x i32> inreg %rsrc, <2 x bfloat> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v2:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
; GFX12PLUS-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX12PLUS-NEXT:    s_wait_loadcnt 0x0
; GFX12PLUS-NEXT:    flat_store_b32 v[1:2], v0
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    s_wait_dscnt 0x0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %out = call <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  store <2 x bfloat> %out, ptr null
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_bf16_1d_v2_noret(<8 x i32> inreg %rsrc, <2 x bfloat> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v2_noret:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %unused = call <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_bf16_1d_v4(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v4:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
; GFX12PLUS-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX12PLUS-NEXT:    s_wait_loadcnt 0x0
; GFX12PLUS-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    s_wait_dscnt 0x0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %out = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  store <4 x bfloat> %out, ptr null
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_noret(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v4_noret:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %unused = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
  ret float 1.0
}

define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_nt(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v4_nt:
; GFX12PLUS:       ; %bb.0: ; %main_body
; GFX12PLUS-NEXT:    image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT
; GFX12PLUS-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX12PLUS-NEXT:    ; return to shader part epilog
main_body:
  %unused = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
  ret float 1.0
}


declare <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half>, i32, <8 x i32>, i32, i32)
declare <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half>, i32, <8 x i32>, i32, i32)
declare <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat>, i32, <8 x i32>, i32, i32)
declare <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat>, i32, <8 x i32>, i32, i32)