; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck --check-prefixes=GFX7 %s
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12 %s

define amdgpu_ps void @store_P0_i8(i8 %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    flat_store_byte v[1:2], v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_i8:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b8 v[1:2], v0
; GFX12-NEXT:    s_endpgm
  store i8 %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P0_i16(i16 %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    flat_store_short v[1:2], v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_i16:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b16 v[1:2], v0
; GFX12-NEXT:    s_endpgm
  store i16 %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P0_i32(i32 %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    flat_store_dword v[1:2], v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b32 v[1:2], v0
; GFX12-NEXT:    s_endpgm
  store i32 %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P0_v2i32(<2 x i32> %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_v2i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    flat_store_dword v[2:3], v0
; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 4, v2
; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX7-NEXT:    flat_store_dword v[2:3], v1
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_v2i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
  store <2 x i32> %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P0_v3i32(<3 x i32> %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_v3i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v3
; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v4, vcc
; GFX7-NEXT:    flat_store_dword v[3:4], v0
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 8, v3
; GFX7-NEXT:    flat_store_dword v[5:6], v1
; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
; GFX7-NEXT:    flat_store_dword v[0:1], v2
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_v3i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b96 v[3:4], v[0:2]
; GFX12-NEXT:    s_endpgm
  store <3 x i32> %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P0_v4i32(<4 x i32> %a, ptr addrspace(0) %out) {
; GFX7-LABEL: store_P0_v4i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_add_i32_e32 v6, vcc, 4, v4
; GFX7-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
; GFX7-NEXT:    flat_store_dword v[4:5], v0
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 8, v4
; GFX7-NEXT:    flat_store_dword v[6:7], v1
; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GFX7-NEXT:    flat_store_dword v[0:1], v2
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 12, v4
; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GFX7-NEXT:    flat_store_dword v[0:1], v3
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P0_v4i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    flat_store_b128 v[4:5], v[0:3]
; GFX12-NEXT:    s_endpgm
  store <4 x i32> %a, ptr addrspace(0) %out
  ret void
}

define amdgpu_ps void @store_P1_i8(i8 %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_byte v0, v[1:2], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_i8:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b8 v[1:2], v0, off
; GFX12-NEXT:    s_endpgm
  store i8 %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P1_i16(i16 %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_short v0, v[1:2], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_i16:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b16 v[1:2], v0, off
; GFX12-NEXT:    s_endpgm
  store i16 %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P1_i32(i32 %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off
; GFX12-NEXT:    s_endpgm
  store i32 %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P1_v2i32(<2 x i32> %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_v2i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_v2i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX12-NEXT:    s_endpgm
  store <2 x i32> %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P1_v3i32(<3 x i32> %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_v3i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_dwordx3 v[0:2], v[3:4], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_v3i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b96 v[3:4], v[0:2], off
; GFX12-NEXT:    s_endpgm
  store <3 x i32> %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P1_v4i32(<4 x i32> %a, ptr addrspace(1) %out) {
; GFX7-LABEL: store_P1_v4i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P1_v4i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b128 v[4:5], v[0:3], off
; GFX12-NEXT:    s_endpgm
  store <4 x i32> %a, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @store_P3_i8(i8 %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b8 v1, v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_i8:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b8 v1, v0
; GFX12-NEXT:    s_endpgm
  store i8 %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P3_i16(i16 %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b16 v1, v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_i16:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  store i16 %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P3_i32(i32 %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b32 v1, v0
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  store i32 %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P3_v2i32(<2 x i32> %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_v2i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b64 v2, v[0:1]
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_v2i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b64 v2, v[0:1]
; GFX12-NEXT:    s_endpgm
  store <2 x i32> %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P3_v3i32(<3 x i32> %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_v3i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b96 v3, v[0:2]
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_v3i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b96 v3, v[0:2]
; GFX12-NEXT:    s_endpgm
  store <3 x i32> %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P3_v4i32(<4 x i32> %a, ptr addrspace(3) %out) {
; GFX7-LABEL: store_P3_v4i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 m0, -1
; GFX7-NEXT:    ds_write_b128 v4, v[0:3]
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P3_v4i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_store_b128 v4, v[0:3]
; GFX12-NEXT:    s_endpgm
  store <4 x i32> %a, ptr addrspace(3) %out
  ret void
}

define amdgpu_ps void @store_P5_i8(i8 %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_byte v0, v1, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_i8:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b8 v1, v0, off
; GFX12-NEXT:    s_endpgm
  store i8 %a, ptr addrspace(5) %out
  ret void
}

define amdgpu_ps void @store_P5_i16(i16 %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_short v0, v1, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_i16:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b16 v1, v0, off
; GFX12-NEXT:    s_endpgm
  store i16 %a, ptr addrspace(5) %out
  ret void
}

define amdgpu_ps void @store_P5_i32(i32 %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b32 v1, v0, off
; GFX12-NEXT:    s_endpgm
  store i32 %a, ptr addrspace(5) %out
  ret void
}

define amdgpu_ps void @store_P5_v2i32(<2 x i32> %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_v2i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_dword v0, v2, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 4, v2
; GFX7-NEXT:    buffer_store_dword v1, v0, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_v2i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT:    s_endpgm
  store <2 x i32> %a, ptr addrspace(5) %out
  ret void
}

define amdgpu_ps void @store_P5_v3i32(<3 x i32> %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_v3i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_dword v0, v3, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 4, v3
; GFX7-NEXT:    buffer_store_dword v1, v0, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 8, v3
; GFX7-NEXT:    buffer_store_dword v2, v0, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_v3i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b96 v3, v[0:2], off
; GFX12-NEXT:    s_endpgm
  store <3 x i32> %a, ptr addrspace(5) %out
  ret void
}

define amdgpu_ps void @store_P5_v4i32(<4 x i32> %a, ptr addrspace(5) %out) {
; GFX7-LABEL: store_P5_v4i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_getpc_b64 s[4:5]
; GFX7-NEXT:    s_mov_b32 s4, s0
; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s4, s4, s0
; GFX7-NEXT:    s_addc_u32 s5, s5, 0
; GFX7-NEXT:    buffer_store_dword v0, v4, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
; GFX7-NEXT:    buffer_store_dword v1, v0, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 8, v4
; GFX7-NEXT:    buffer_store_dword v2, v0, s[4:7], 0 offen
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 12, v4
; GFX7-NEXT:    buffer_store_dword v3, v0, s[4:7], 0 offen
; GFX7-NEXT:    s_endpgm
;
; GFX12-LABEL: store_P5_v4i32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    scratch_store_b128 v4, v[0:3], off
; GFX12-NEXT:    s_endpgm
  store <4 x i32> %a, ptr addrspace(5) %out
  ret void
}