derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
2185 lines
141 KiB
LLVM
2185 lines
141 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX7 %s
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX900 %s
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX908 %s
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX90A %s
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX942 %s
|
|
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck -check-prefixes=ALL,GFX12 %s
|
|
|
|
; --------------------------------------------------------------------
|
|
; Idempotent expansion cases without noalias.addrspace
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_add_0_i64_agent(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_add_0_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 0
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw add ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_0_i64_agent(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_0_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 0
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_or_0_i64_agent(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_or_0_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 0
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw or ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_xor_0_i64_agent(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_xor_0_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 0
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw xor ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; Idempotent expansion cases with noalias.addrspace
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_add_0_i64_agent__noalias_addrspace_5(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_add_0_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw add ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_0_i64_agent__noalias_addrspace_5(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_0_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_or_0_i64_agent__noalias_addrspace_5(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_or_0_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw or ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw or ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_xor_0_i64_agent__noalias_addrspace_5(ptr %ptr) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_xor_0_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw xor ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw xor ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for add
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_add_i64_agent(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_add_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], [[VALUE]]
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw add ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_add_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_add_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw add ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i32 @test_flat_atomicrmw_add_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {
|
|
; ALL-LABEL: define i32 @test_flat_atomicrmw_add_i32_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i32 [[RES]]
|
|
;
|
|
%res = atomicrmw add ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for xchg
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_xchg_i64_agent(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_xchg_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: store i64 [[VALUE]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_xchg_i64_agent__noalias_xchgrspace_5(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_xchg_i64_agent__noalias_xchgrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i32 @test_flat_atomicrmw_xchg_i32_agent__noalias_xchgrspace_5(ptr %ptr, i32 %value) {
|
|
; ALL-LABEL: define i32 @test_flat_atomicrmw_xchg_i32_agent__noalias_xchgrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i32 [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for xchg (pointer type)
|
|
; --------------------------------------------------------------------
|
|
|
|
define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent(ptr %ptr, ptr addrspace(1) %value) {
|
|
; ALL-LABEL: define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(1) [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: store ptr addrspace(1) [[VALUE]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(1) [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi ptr addrspace(1) [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret ptr addrspace(1) [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, ptr addrspace(1) %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret ptr addrspace(1) %res
|
|
}
|
|
|
|
define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent__noalias_xchgrspace_5(ptr %ptr, ptr addrspace(1) %value) {
|
|
; ALL-LABEL: define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent__noalias_xchgrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(1) [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(1) [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret ptr addrspace(1) [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, ptr addrspace(1) %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret ptr addrspace(1) %res
|
|
}
|
|
|
|
define ptr addrspace(3) @test_flat_atomicrmw_xchg_p3_agent__noalias_xchgrspace_5(ptr %ptr, ptr addrspace(3) %value) {
|
|
; ALL-LABEL: define ptr addrspace(3) @test_flat_atomicrmw_xchg_p3_agent__noalias_xchgrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(3) [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(3) [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret ptr addrspace(3) [[RES]]
|
|
;
|
|
%res = atomicrmw xchg ptr %ptr, ptr addrspace(3) %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret ptr addrspace(3) %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for and
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_and_i64_agent(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = and i64 [[LOADED_PRIVATE]], [[VALUE]]
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {
|
|
; GFX7-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX7-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX900-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX900-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX908-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX90A-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX940-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX940-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX940-NEXT: ret i64 [[RES]]
|
|
;
|
|
; GFX12-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX12-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret i64 %res
|
|
}
|
|
|
|
define i32 @test_flat_atomicrmw_and_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {
|
|
; ALL-LABEL: define i32 @test_flat_atomicrmw_and_i32_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i32 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_and_i64_agent__mmra(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__mmra(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = and i64 [[LOADED_PRIVATE]], [[VALUE]]
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2:![0-9]+]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !mmra !4, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__mmra(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__mmra(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !mmra !4, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for subb
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_sub_i64_agent(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = sub i64 [[LOADED_PRIVATE]], [[VALUE]]
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {
|
|
; GFX7-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX7-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX900-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX900-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX908-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX90A-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret i64 [[NEWLOADED]]
|
|
;
|
|
; GFX940-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX940-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX940-NEXT: ret i64 [[RES]]
|
|
;
|
|
; GFX12-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX12-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret i64 %res
|
|
}
|
|
|
|
define i32 @test_flat_atomicrmw_sub_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {
|
|
; ALL-LABEL: define i32 @test_flat_atomicrmw_sub_i32_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i32 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_i64_agent__mmra(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__mmra(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[NEW:%.*]] = sub i64 [[LOADED_PRIVATE]], [[VALUE]]
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP2:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !mmra !4, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__mmra(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__mmra(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !mmra !4, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for fadd
|
|
; --------------------------------------------------------------------
|
|
|
|
define double @test_flat_atomicrmw_fadd_f64_agent(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX7: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX7-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX7: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX7-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
|
|
; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END1]]:
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX7: [[ATOMICRMW_PHI]]:
|
|
; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX900: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX900-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX900: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX900-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX900-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END1]]:
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX900: [[ATOMICRMW_PHI]]:
|
|
; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX908: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX908: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END1]]:
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX908: [[ATOMICRMW_PHI]]:
|
|
; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]
|
|
; GFX90A: [[ATOMICRMW_SHARED]]:
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX90A: [[ATOMICRMW_CHECK_PRIVATE]]:
|
|
; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX90A: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
|
|
; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_PHI]]:
|
|
; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
|
|
; GFX942-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]
|
|
; GFX942: [[ATOMICRMW_SHARED]]:
|
|
; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
|
|
; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX942: [[ATOMICRMW_CHECK_PRIVATE]]:
|
|
; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX942: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX942: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
|
|
; GFX942-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX942: [[ATOMICRMW_PHI]]:
|
|
; GFX942-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX942: [[ATOMICRMW_END]]:
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX12: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX12-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX12: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX12-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX12-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END1]]:
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX12: [[ATOMICRMW_PHI]]:
|
|
; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[RES]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX7-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX7-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret double %res
|
|
}
|
|
|
|
define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {
|
|
; GFX7-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX900-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]
|
|
; GFX90A: [[ATOMICRMW_SHARED]]:
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX90A: [[ATOMICRMW_CHECK_PRIVATE]]:
|
|
; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX90A: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load float, ptr addrspace(5) [[TMP3]], align 4
|
|
; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED_PRIVATE]], [[VALUE]]
|
|
; GFX90A-NEXT: store float [[NEW]], ptr addrspace(5) [[TMP3]], align 4
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
|
|
; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_PHI]]:
|
|
; GFX90A-NEXT: [[LOADED_PHI:%.*]] = phi float [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret float [[LOADED_PHI]]
|
|
;
|
|
; GFX942-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]
|
|
; GFX942-NEXT: ret float [[RES]]
|
|
;
|
|
; GFX12-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]
|
|
; GFX12-NEXT: ret float [[RES]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
|
|
ret float %res
|
|
}
|
|
|
|
define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(ptr %ptr, <2 x half> %value) {
|
|
; GFX7-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 4
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i32
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i32
|
|
; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret <2 x half> [[TMP5]]
|
|
;
|
|
; GFX900-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 4
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i32
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i32
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret <2 x half> [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 4
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i32
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i32
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret <2 x half> [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 4
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i32
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i32
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret <2 x half> [[TMP5]]
|
|
;
|
|
; GFX942-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x half> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: ret <2 x half> [[RES]]
|
|
;
|
|
; GFX12-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x half> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: ret <2 x half> [[RES]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret <2 x half> %res
|
|
}
|
|
|
|
define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(ptr %ptr, <2 x bfloat> %value) {
|
|
; GFX7-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 4
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i32
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i32
|
|
; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret <2 x bfloat> [[TMP5]]
|
|
;
|
|
; GFX900-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 4
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i32
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i32
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret <2 x bfloat> [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 4
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i32
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i32
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 4
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i32
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i32
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]
|
|
;
|
|
; GFX942-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x bfloat> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: ret <2 x bfloat> [[RES]]
|
|
;
|
|
; GFX12-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x bfloat> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: ret <2 x bfloat> [[RES]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret <2 x bfloat> %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for fmin
|
|
; --------------------------------------------------------------------
|
|
|
|
define double @test_flat_atomicrmw_fmin_f64_agent(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX7: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX7-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX7: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX7-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX7: [[ATOMICRMW_PHI]]:
|
|
; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX900: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX900-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX900: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX900-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX900-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX900-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END1]]:
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX900: [[ATOMICRMW_PHI]]:
|
|
; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX908: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX908-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX908: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX908-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX908-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX908-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END1]]:
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX908: [[ATOMICRMW_PHI]]:
|
|
; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX90A: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX90A-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX90A: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_PHI]]:
|
|
; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX942: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX942-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX942: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX942-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX942: [[ATOMICRMW_PHI]]:
|
|
; GFX942-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX942: [[ATOMICRMW_END]]:
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX12: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX12-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX12: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX12-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX12-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX12-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END1]]:
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX12: [[ATOMICRMW_PHI]]:
|
|
; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[RES]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
|
|
; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[TMP6]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret double %res
|
|
}
|
|
|
|
define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {
|
|
; GFX7-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: ret float [[RES]]
|
|
;
|
|
; GFX900-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[LOADED_PHI:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX90A-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; GFX90A-NEXT: [[LOADED_PHI]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret float [[LOADED_PHI]]
|
|
;
|
|
; GFX942-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX942: [[ATOMICRMW_START]]:
|
|
; GFX942-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX942-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX942-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; GFX942-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX942-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; GFX942-NEXT: [[RES]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX942-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX942: [[ATOMICRMW_END]]:
|
|
; GFX942-NEXT: ret float [[RES]]
|
|
;
|
|
; GFX12-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: ret float [[RES]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret float %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for fmax
|
|
; --------------------------------------------------------------------
|
|
|
|
define double @test_flat_atomicrmw_fmax_f64_agent(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX7: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX7-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX7: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX7-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX7: [[ATOMICRMW_PHI]]:
|
|
; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX900: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX900-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX900: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX900-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX900-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX900-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END1]]:
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX900: [[ATOMICRMW_PHI]]:
|
|
; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX908: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX908-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX908: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX908-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX908-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX908-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END1]]:
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX908: [[ATOMICRMW_PHI]]:
|
|
; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX90A: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX90A-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX90A: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX90A: [[ATOMICRMW_PHI]]:
|
|
; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX942: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX942-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX942: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX942-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX942: [[ATOMICRMW_PHI]]:
|
|
; GFX942-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX942: [[ATOMICRMW_END]]:
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; GFX12: [[ATOMICRMW_PRIVATE]]:
|
|
; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])
|
|
; GFX12-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; GFX12: [[ATOMICRMW_GLOBAL]]:
|
|
; GFX12-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i64
|
|
; GFX12-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 0
|
|
; GFX12-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END1]]:
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; GFX12: [[ATOMICRMW_PHI]]:
|
|
; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[RES]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret double %res
|
|
}
|
|
|
|
define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {
|
|
; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX7: [[ATOMICRMW_START]]:
|
|
; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
|
|
; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX7: [[ATOMICRMW_END]]:
|
|
; GFX7-NEXT: ret double [[TMP6]]
|
|
;
|
|
; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret double [[TMP5]]
|
|
;
|
|
; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX942-NEXT: ret double [[RES]]
|
|
;
|
|
; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8
|
|
; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX12: [[ATOMICRMW_START]]:
|
|
; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
|
|
; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX12: [[ATOMICRMW_END]]:
|
|
; GFX12-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret double %res
|
|
}
|
|
|
|
define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {
|
|
; GFX7-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX7-NEXT: ret float [[RES]]
|
|
;
|
|
; GFX900-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX900: [[ATOMICRMW_START]]:
|
|
; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX900-NEXT: [[NEW:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX900: [[ATOMICRMW_END]]:
|
|
; GFX900-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX908-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX908: [[ATOMICRMW_START]]:
|
|
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX908-NEXT: [[NEW:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX908: [[ATOMICRMW_END]]:
|
|
; GFX908-NEXT: ret float [[TMP5]]
|
|
;
|
|
; GFX90A-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX90A: [[ATOMICRMW_START]]:
|
|
; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[LOADED_PHI:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX90A-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; GFX90A-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX90A-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; GFX90A-NEXT: [[LOADED_PHI]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX90A: [[ATOMICRMW_END]]:
|
|
; GFX90A-NEXT: ret float [[LOADED_PHI]]
|
|
;
|
|
; GFX942-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX942-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4
|
|
; GFX942-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; GFX942: [[ATOMICRMW_START]]:
|
|
; GFX942-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; GFX942-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])
|
|
; GFX942-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; GFX942-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; GFX942-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; GFX942-NEXT: [[RES]] = bitcast i32 [[NEWLOADED]] to float
|
|
; GFX942-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; GFX942: [[ATOMICRMW_END]]:
|
|
; GFX942-NEXT: ret float [[RES]]
|
|
;
|
|
; GFX12-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(
|
|
; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; GFX12-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; GFX12-NEXT: ret float [[RES]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret float %res
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; General expansion for nand
|
|
; --------------------------------------------------------------------
|
|
|
|
define i64 @test_flat_atomicrmw_nand_i64_agent(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
|
|
; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
|
|
; ALL: [[ATOMICRMW_PRIVATE]]:
|
|
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
|
|
; ALL-NEXT: [[LOADED:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1
|
|
; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]
|
|
; ALL: [[ATOMICRMW_GLOBAL]]:
|
|
; ALL-NEXT: [[TMP3:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; ALL: [[ATOMICRMW_START]]:
|
|
; ALL-NEXT: [[LOADED1:%.*]] = phi i64 [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
|
|
; ALL-NEXT: [[TMP4:%.*]] = and i64 [[LOADED1]], [[VALUE]]
|
|
; ALL-NEXT: [[NEW2:%.*]] = xor i64 [[TMP4]], -1
|
|
; ALL-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED1]], i64 [[NEW2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; ALL-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]
|
|
; ALL: [[ATOMICRMW_END1]]:
|
|
; ALL-NEXT: br label %[[ATOMICRMW_PHI]]
|
|
; ALL: [[ATOMICRMW_PHI]]:
|
|
; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED]], %[[ATOMICRMW_PRIVATE]] ], [ [[NEWLOADED]], %[[ATOMICRMW_END1]] ]
|
|
; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; ALL: [[ATOMICRMW_START]]:
|
|
; ALL-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1
|
|
; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
|
|
; ALL-NEXT: [[RES]] = extractvalue { i64, i1 } [[TMP3]], 0
|
|
; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i64 %res
|
|
}
|
|
|
|
define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {
|
|
; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5__maybe_fine_grained(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
|
|
; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; ALL: [[ATOMICRMW_START]]:
|
|
; ALL-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]
|
|
; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -1
|
|
; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
|
|
; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
|
|
; ALL-NEXT: [[RES]] = extractvalue { i64, i1 } [[TMP3]], 0
|
|
; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i64 [[RES]]
|
|
;
|
|
%res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1
|
|
ret i64 %res
|
|
}
|
|
|
|
|
|
define i32 @test_flat_atomicrmw_nand_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {
|
|
; ALL-LABEL: define i32 @test_flat_atomicrmw_nand_i32_agent__noalias_addrspace_5(
|
|
; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
|
|
; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR]], align 4
|
|
; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]
|
|
; ALL: [[ATOMICRMW_START]]:
|
|
; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]
|
|
; ALL-NEXT: [[TMP2:%.*]] = and i32 [[LOADED]], [[VALUE]]
|
|
; ALL-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -1
|
|
; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]
|
|
; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
|
|
; ALL-NEXT: [[RES]] = extractvalue { i32, i1 } [[TMP3]], 0
|
|
; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
|
|
; ALL: [[ATOMICRMW_END]]:
|
|
; ALL-NEXT: ret i32 [[RES]]
|
|
;
|
|
%res = atomicrmw nand ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %res
|
|
}
|
|
|
|
!0 = !{}
|
|
!1 = !{i32 5, i32 6}
|
|
!2 = !{!"foo", !"bar"}
|
|
!3 = !{!"bux", !"baz"}
|
|
!4 = !{!2, !3}
|
|
!5 = !{}
|
|
|
|
;.
|
|
; GFX7: [[META0]] = !{i32 5, i32 6}
|
|
; GFX7: [[META1]] = !{}
|
|
; GFX7: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX7: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX7: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|
|
; GFX900: [[META0]] = !{i32 5, i32 6}
|
|
; GFX900: [[META1]] = !{}
|
|
; GFX900: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX900: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX900: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|
|
; GFX908: [[META0]] = !{i32 5, i32 6}
|
|
; GFX908: [[META1]] = !{}
|
|
; GFX908: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX908: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX908: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|
|
; GFX90A: [[META0]] = !{i32 5, i32 6}
|
|
; GFX90A: [[META1]] = !{}
|
|
; GFX90A: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX90A: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX90A: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|
|
; GFX942: [[META0]] = !{i32 5, i32 6}
|
|
; GFX942: [[META1]] = !{}
|
|
; GFX942: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX942: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX942: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|
|
; GFX12: [[META0]] = !{i32 5, i32 6}
|
|
; GFX12: [[META1]] = !{}
|
|
; GFX12: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}
|
|
; GFX12: [[META3]] = !{!"foo", !"bar"}
|
|
; GFX12: [[META4]] = !{!"bux", !"baz"}
|
|
;.
|