; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX803 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX906 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX908 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX90A %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX942 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX10 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX11 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -passes='require,atomic-expand' %s | FileCheck -check-prefixes=COMMON,GFX12 %s ;--------------------------------------------------------------------- ; atomicrmw xchg ;--------------------------------------------------------------------- ; xchg is supported over PCIe, so no expansion is necessary define i64 @test_atomicrmw_xchg_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xchg_i64_global_agent( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw xchg ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } ; xchg is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw xchg ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } ; xchg is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw xchg ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } ; xchg is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xchg_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw xchg ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw add ;--------------------------------------------------------------------- ; add is supported over PCIe, so no expansion is necessary define i64 @test_atomicrmw_add_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_add_i64_global_agent( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw add ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } ; add is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw add ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } ; add is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw add ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } ; add is supported over PCIe, so no expansion is necessary. Metadata should be ignored. define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[RES]] ; %res = atomicrmw add ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw sub ;--------------------------------------------------------------------- ; expansion is necessary, sub is not supported over PCIe define i64 @test_atomicrmw_sub_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw and ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_and_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_and_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_and_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw nand ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported define i64 @test_atomicrmw_nand_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_nand_i64_global_agent( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1 ; COMMON-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw nand ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1 ; COMMON-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw nand ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1 ; COMMON-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw nand ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1 ; COMMON-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw nand ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw or ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_or_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_or_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw xor ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_xor_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw max ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_max_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX803-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX906-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX908-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX90A-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX10-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX11-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_max_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw min ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_min_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX803-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX906-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX908-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX90A-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX10-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX11-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_min_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw umax ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_umax_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX803-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX906-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX908-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX90A-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX10-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX11-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw umin ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_umin_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX803-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX906-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX908-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX90A-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX10-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]] ; GFX11-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw uinc_wrap ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX803-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX803-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX906-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX906-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX908-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX90A-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX10-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX10-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = add i64 [[LOADED]], 1 ; GFX11-NEXT: [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]] ; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw udec_wrap ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_udec_wrap_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; GFX803-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX803-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX803-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX803: atomicrmw.start: ; GFX803-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX803-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX803-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX803-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX803-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX803-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX803-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX803-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX803-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX803-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX803: atomicrmw.end: ; GFX803-NEXT: ret i64 [[NEWLOADED]] ; ; GFX906-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX906-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX906-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX906: atomicrmw.start: ; GFX906-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX906-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX906-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX906-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX906-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX906-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX906-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX906-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX906-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX906-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX906: atomicrmw.end: ; GFX906-NEXT: ret i64 [[NEWLOADED]] ; ; GFX908-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908: atomicrmw.start: ; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX908-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX908-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX908-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX908-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX908-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908: atomicrmw.end: ; GFX908-NEXT: ret i64 [[NEWLOADED]] ; ; GFX90A-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX90A: atomicrmw.start: ; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX90A-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX90A-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX90A-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX90A-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX90A-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX90A-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX90A: atomicrmw.end: ; GFX90A-NEXT: ret i64 [[NEWLOADED]] ; ; GFX942-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX942-NEXT: [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX942-NEXT: ret i64 [[RES]] ; ; GFX10-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX10-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX10-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX10: atomicrmw.start: ; GFX10-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX10-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX10-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX10-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX10-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX10-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX10-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX10-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX10-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX10: atomicrmw.end: ; GFX10-NEXT: ret i64 [[NEWLOADED]] ; ; GFX11-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX11-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX11: atomicrmw.start: ; GFX11-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; GFX11-NEXT: [[TMP2:%.*]] = sub i64 [[LOADED]], 1 ; GFX11-NEXT: [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0 ; GFX11-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]] ; GFX11-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; GFX11-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]] ; GFX11-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1 ; GFX11-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0 ; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX11: atomicrmw.end: ; GFX11-NEXT: ret i64 [[NEWLOADED]] ; ; GFX12-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX12-NEXT: [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX12-NEXT: ret i64 [[RES]] ; ; GFX940-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent( ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; GFX940-NEXT: [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8 ; GFX940-NEXT: ret i64 [[RES]] %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw usub_cond ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_usub_cond_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_cond_i64_global_agent( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[TMP3:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[TMP3]], i64 [[LOADED]] ; COMMON-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_cond ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[TMP3:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[TMP3]], i64 [[LOADED]] ; COMMON-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_cond ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[TMP3:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[TMP3]], i64 [[LOADED]] ; COMMON-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_cond ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_cond_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[TMP2:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[TMP3:%.*]] = sub i64 [[LOADED]], [[VALUE]] ; COMMON-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i64 [[TMP3]], i64 [[LOADED]] ; COMMON-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_cond ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } ;--------------------------------------------------------------------- ; atomicrmw usub_sat ;--------------------------------------------------------------------- ; expansion is necessary, operation not supported over PCIe define i64 @test_atomicrmw_usub_sat_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_sat_i64_global_agent( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[NEW:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[LOADED]], i64 [[VALUE]]) ; COMMON-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8 ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_sat ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst ret i64 %res } define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_fine_grained_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[NEW:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[LOADED]], i64 [[VALUE]]) ; COMMON-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_sat ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0 ret i64 %res } define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[NEW:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[LOADED]], i64 [[VALUE]]) ; COMMON-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_sat ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0 ret i64 %res } define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) { ; COMMON-LABEL: define i64 @test_atomicrmw_usub_sat_i64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory( ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] { ; COMMON-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8 ; COMMON-NEXT: br label [[ATOMICRMW_START:%.*]] ; COMMON: atomicrmw.start: ; COMMON-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ] ; COMMON-NEXT: [[NEW:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[LOADED]], i64 [[VALUE]]) ; COMMON-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]] ; COMMON-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1 ; COMMON-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0 ; COMMON-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; COMMON: atomicrmw.end: ; COMMON-NEXT: ret i64 [[NEWLOADED]] ; %res = atomicrmw usub_sat ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 ret i64 %res } !0 = !{} ;. ; GFX803: [[META0]] = !{} ;. ; GFX906: [[META0]] = !{} ;. ; GFX908: [[META0]] = !{} ;. ; GFX90A: [[META0]] = !{} ;. ; GFX942: [[META0]] = !{} ;. ; GFX10: [[META0]] = !{} ;. ; GFX11: [[META0]] = !{} ;. ; GFX12: [[META0]] = !{} ;.