; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s

; Test atomicrmw xchg operations for different address spaces

; =============================================================================
; atomicrmw xchg - global address space (addrspace 1)
; =============================================================================

define i32 @atomicrmw_xchg_i32_global(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_xchg_i32_global:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_xchg_i64_global(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_xchg_i64_global:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i64 %val seq_cst
  ret i64 %result
}

; =============================================================================
; atomicrmw xchg - local address space (addrspace 3)
; =============================================================================

define i32 @atomicrmw_xchg_i32_local(ptr addrspace(3) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_xchg_i32_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(3) %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_xchg_i64_local(ptr addrspace(3) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_xchg_i64_local:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    ds_storexchg_rtn_b64 v[0:1], v0, v[1:2]
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(3) %ptr, i64 %val seq_cst
  ret i64 %result
}

; =============================================================================
; atomicrmw xchg - flat address space (addrspace 0)
; =============================================================================

define i32 @atomicrmw_xchg_i32_flat(ptr %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_xchg_i32_flat:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_swap_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, i32 %val seq_cst
  ret i32 %result
}

define i64 @atomicrmw_xchg_i64_flat(ptr %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_xchg_i64_flat:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, i64 %val seq_cst, !noalias.addrspace !1
  ret i64 %result
}

; =============================================================================
; atomicrmw xchg - VGPR inputs (loaded from memory)
; =============================================================================

define i32 @atomicrmw_xchg_i32_global_vgpr(ptr addrspace(1) %ptr, ptr addrspace(1) %val_ptr) {
; GFX12-LABEL: atomicrmw_xchg_i32_global_vgpr:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v2, v[2:3], off
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %val = load i32, ptr addrspace(1) %val_ptr
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %val seq_cst
  ret i32 %result
}

; =============================================================================
; atomicrmw xchg with metadata - global address space
; Note: xchg is natively supported and doesn't expand to CAS for i32/i64,
; but we still test metadata for consistency and documentation purposes
; =============================================================================

define i32 @atomicrmw_xchg_i32_global_no_remote_memory(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_xchg_i32_global_no_remote_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %val seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

define i64 @atomicrmw_xchg_i64_global_no_remote_memory(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_xchg_i64_global_no_remote_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i64 %val seq_cst, !amdgpu.no.remote.memory !0
  ret i64 %result
}

define i32 @atomicrmw_xchg_i32_global_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %val) {
; GFX12-LABEL: atomicrmw_xchg_i32_global_no_fine_grained_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %val seq_cst, !amdgpu.no.fine.grained.memory !0
  ret i32 %result
}

define i64 @atomicrmw_xchg_i64_global_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %val) {
; GFX12-LABEL: atomicrmw_xchg_i64_global_no_fine_grained_memory:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    s_wait_expcnt 0x0
; GFX12-NEXT:    s_wait_samplecnt 0x0
; GFX12-NEXT:    s_wait_bvhcnt 0x0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    global_atomic_swap_b64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr addrspace(1) %ptr, i64 %val seq_cst, !amdgpu.no.fine.grained.memory !0
  ret i64 %result
}

!0 = !{}
!1 = !{i32 5, i32 6}  ; Exclude private address space (5) to prevent expansion