derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
256 lines
11 KiB
LLVM
256 lines
11 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefix=GFX942
|
|
|
|
; =============================================================================
|
|
; Flat atomic fadd - f32
|
|
; =============================================================================
|
|
|
|
define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat(ptr %ptr) {
|
|
; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
|
|
; GFX942-NEXT: v_mov_b32_e32 v2, 4.0
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
|
|
; GFX942-NEXT: buffer_wbl2 sc0 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc0 sc1
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat_ieee(ptr %ptr) #0 {
|
|
; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
|
|
; GFX942-NEXT: v_mov_b32_e32 v2, 4.0
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
|
|
; GFX942-NEXT: buffer_wbl2 sc0 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc0 sc1
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define float @flat_atomic_fadd_f32_rtn_pat(ptr %ptr, float %data) {
|
|
; GFX942-LABEL: flat_atomic_fadd_f32_rtn_pat:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: v_mov_b32_e32 v2, 4.0
|
|
; GFX942-NEXT: buffer_wbl2 sc0 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc0 sc1
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0
|
|
ret float %ret
|
|
}
|
|
|
|
; =============================================================================
|
|
; Flat atomic fadd - v2f16
|
|
; =============================================================================
|
|
|
|
define <2 x half> @flat_atomic_fadd_ret_v2f16_agent_offset(ptr %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: flat_atomic_fadd_ret_v2f16_agent_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_wbl2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:1024 sc0
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc1
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
|
|
%result = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
|
|
ret <2 x half> %result
|
|
}
|
|
|
|
define void @flat_atomic_fadd_noret_v2f16_agent_offset(ptr %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: flat_atomic_fadd_noret_v2f16_agent_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_wbl2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: flat_atomic_pk_add_f16 v[0:1], v2 offset:1024
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc1
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
|
|
%unused = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
|
|
ret void
|
|
}
|
|
|
|
; =============================================================================
|
|
; Global atomic fadd - f32
|
|
; =============================================================================
|
|
|
|
define amdgpu_ps void @global_atomic_fadd_f32_no_rtn_atomicrmw(ptr addrspace(1) %ptr, float %data) {
|
|
; GFX942-LABEL: global_atomic_fadd_f32_no_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: global_atomic_add_f32 v[0:1], v2, off
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps float @global_atomic_fadd_f32_rtn_atomicrmw(ptr addrspace(1) %ptr, float %data) {
|
|
; GFX942-LABEL: global_atomic_fadd_f32_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: global_atomic_add_f32 v0, v[0:1], v2, off sc0
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: ; return to shader part epilog
|
|
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret float %ret
|
|
}
|
|
|
|
; =============================================================================
|
|
; Global atomic fadd - v2f16
|
|
; =============================================================================
|
|
|
|
define <2 x half> @global_atomic_fadd_ret_v2f16_agent_offset(ptr addrspace(1) %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: global_atomic_fadd_ret_v2f16_agent_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_wbl2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: global_atomic_pk_add_f16 v0, v[0:1], v2, off offset:1024 sc0
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc1
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i32 256
|
|
%result = atomicrmw fadd ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
|
|
ret <2 x half> %result
|
|
}
|
|
|
|
define void @global_atomic_fadd_noret_v2f16_agent_offset(ptr addrspace(1) %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: global_atomic_fadd_noret_v2f16_agent_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: buffer_wbl2 sc1
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: global_atomic_pk_add_f16 v[0:1], v2, off offset:1024
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: buffer_inv sc1
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr <2 x half>, ptr addrspace(1) %ptr, i32 256
|
|
%unused = atomicrmw fadd ptr addrspace(1) %gep, <2 x half> %val syncscope("agent") seq_cst
|
|
ret void
|
|
}
|
|
|
|
; =============================================================================
|
|
; Global atomic fadd - f64
|
|
; =============================================================================
|
|
|
|
define amdgpu_ps void @global_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(1) %ptr, double %data) {
|
|
; GFX942-LABEL: global_atomic_fadd_f64_no_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: global_atomic_add_f64 v[0:1], v[2:3], off
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %ptr, double %data) {
|
|
; GFX942-LABEL: global_atomic_fadd_f64_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX942-NEXT: v_readfirstlane_b32 s0, v0
|
|
; GFX942-NEXT: v_readfirstlane_b32 s1, v1
|
|
; GFX942-NEXT: ; return to shader part epilog
|
|
%ret = atomicrmw fadd ptr addrspace(1) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret double %ret
|
|
}
|
|
|
|
; =============================================================================
|
|
; Local atomic fadd - f32
|
|
; =============================================================================
|
|
|
|
define amdgpu_ps void @local_atomic_fadd_f32_no_rtn_atomicrmw(ptr addrspace(3) %ptr, float %data) {
|
|
; GFX942-LABEL: local_atomic_fadd_f32_no_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: ds_add_f32 v0, v1
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr addrspace(3) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps float @local_atomic_fadd_f32_rtn_atomicrmw(ptr addrspace(3) %ptr, float %data) {
|
|
; GFX942-LABEL: local_atomic_fadd_f32_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: ds_add_rtn_f32 v0, v0, v1
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: ; return to shader part epilog
|
|
%ret = atomicrmw fadd ptr addrspace(3) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret float %ret
|
|
}
|
|
|
|
; =============================================================================
|
|
; Local atomic fadd - v2f16
|
|
; =============================================================================
|
|
|
|
define <2 x half> @local_atomic_fadd_ret_v2f16_offset(ptr addrspace(3) %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: local_atomic_fadd_ret_v2f16_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: ds_pk_add_rtn_f16 v0, v0, v1 offset:65532
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
|
|
%result = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst
|
|
ret <2 x half> %result
|
|
}
|
|
|
|
define void @local_atomic_fadd_noret_v2f16_offset(ptr addrspace(3) %ptr, <2 x half> %val) {
|
|
; GFX942-LABEL: local_atomic_fadd_noret_v2f16_offset:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX942-NEXT: ds_pk_add_f16 v0, v1 offset:65532
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 16383
|
|
%unused = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst
|
|
ret void
|
|
}
|
|
|
|
; =============================================================================
|
|
; Local atomic fadd - f64
|
|
; =============================================================================
|
|
|
|
define amdgpu_ps void @local_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(3) %ptr, double %data) {
|
|
; GFX942-LABEL: local_atomic_fadd_f64_no_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: v_mov_b32_e32 v4, v1
|
|
; GFX942-NEXT: v_mov_b32_e32 v5, v2
|
|
; GFX942-NEXT: ds_add_f64 v0, v[4:5]
|
|
; GFX942-NEXT: s_endpgm
|
|
%ret = atomicrmw fadd ptr addrspace(3) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps double @local_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(3) %ptr, double %data) {
|
|
; GFX942-LABEL: local_atomic_fadd_f64_rtn_atomicrmw:
|
|
; GFX942: ; %bb.0:
|
|
; GFX942-NEXT: v_mov_b32_e32 v4, v1
|
|
; GFX942-NEXT: v_mov_b32_e32 v5, v2
|
|
; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[4:5]
|
|
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GFX942-NEXT: v_readfirstlane_b32 s0, v0
|
|
; GFX942-NEXT: v_readfirstlane_b32 s1, v1
|
|
; GFX942-NEXT: ; return to shader part epilog
|
|
%ret = atomicrmw fadd ptr addrspace(3) %ptr, double %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
|
|
ret double %ret
|
|
}
|
|
|
|
attributes #0 = { denormal_fpenv(float: ieee|ieee) }
|
|
|
|
!0 = !{}
|