; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s

; Test atomicrmw add and sub operations for different address spaces

; =============================================================================
; atomicrmw add - global address space (addrspace 1)
; =============================================================================

define i32 @atomicrmw_add_i32_global(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_add_i32_global:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_add_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr addrspace(1) %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_add_i64_global(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_add_i64_global:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_add_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr addrspace(1) %ptr, i64 %val seq_cst
  ret i64 %result
}

; =============================================================================
; atomicrmw add - local address space (addrspace 3)
; =============================================================================

define i32 @atomicrmw_add_i32_local(ptr addrspace(3) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_add_i32_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_add_rtn_u32 v0, v0, v1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr addrspace(3) %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_add_i64_local(ptr addrspace(3) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_add_i64_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_add_rtn_u64 v[0:1], v0, v[1:2]
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr addrspace(3) %ptr, i64 %val seq_cst
  ret i64 %result
}

; =============================================================================
; atomicrmw sub - local address space (addrspace 3)
; =============================================================================

define i32 @atomicrmw_sub_i32_local(ptr addrspace(3) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_sub_i32_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_sub_rtn_u32 v0, v0, v1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(3) %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_sub_i64_local(ptr addrspace(3) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_sub_i64_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_sub_rtn_u64 v[0:1], v0, v[1:2]
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(3) %ptr, i64 %val seq_cst
  ret i64 %result
}

; =============================================================================
; atomicrmw add - flat address space (addrspace 0)
; =============================================================================

define i32 @atomicrmw_add_i32_flat(ptr %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_add_i32_flat:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_add_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_add_i64_flat(ptr %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_add_i64_flat:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr %ptr, i64 %val seq_cst, !noalias.addrspace !1
  ret i64 %result
}

; =============================================================================
; atomicrmw add - VGPR inputs (loaded from memory)
; =============================================================================

define i32 @atomicrmw_add_i32_global_vgpr(ptr addrspace(1) %ptr, ptr addrspace(1) %val_ptr) {
; GFX12-LABEL: atomicrmw_add_i32_global_vgpr:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v2, v[2:3], off
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_add_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %val = load i32, ptr addrspace(1) %val_ptr
  %result = atomicrmw add ptr addrspace(1) %ptr, i32 %val seq_cst
  ret i32 %result
}

; =============================================================================
; atomicrmw sub with metadata - global address space (no expansion)
; =============================================================================

define i32 @atomicrmw_sub_i32_global_no_remote_memory(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_sub_i32_global_no_remote_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_sub_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %val seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

define i64 @atomicrmw_sub_i64_global_no_remote_memory(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_sub_i64_global_no_remote_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_sub_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %val seq_cst, !amdgpu.no.remote.memory !0
  ret i64 %result
}

define i32 @atomicrmw_sub_i32_global_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_sub_i32_global_no_fine_grained_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_sub_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %val seq_cst, !amdgpu.no.fine.grained.memory !0
  ret i32 %result
}

define i64 @atomicrmw_sub_i64_global_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_sub_i64_global_no_fine_grained_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_sub_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %val seq_cst, !amdgpu.no.fine.grained.memory !0
  ret i64 %result
}

!0 = !{}
!1 = !{i32 5, i32 6}  ; Exclude private address space (5) to prevent expansion