derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
575 lines
25 KiB
LLVM
575 lines
25 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=SDAG
|
|
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GISEL
|
|
|
|
; Test async mark/wait with global_load_lds and global loads
|
|
; This version uses wave barriers to enforce program order so that unrelated vmem
|
|
; instructions do not get reordered before reaching this point.
|
|
|
|
define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
|
|
; SDAG-LABEL: interleaved_with_wave_barrier:
|
|
; SDAG: ; %bb.0: ; %entry
|
|
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; SDAG-NEXT: s_wait_kmcnt 0x0
|
|
; SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
|
|
; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
|
|
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
|
|
; SDAG-NEXT: global_load_b32 v10, v[8:9], off offset:44
|
|
; SDAG-NEXT: global_load_b32 v11, v[0:1], off offset:4
|
|
; SDAG-NEXT: ; wave barrier
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
|
|
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
|
|
; SDAG-NEXT: ; wave barrier
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
|
|
; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
|
|
; SDAG-NEXT: ; wave barrier
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
|
|
; SDAG-NEXT: ; wave barrier
|
|
; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: ; wait_asyncmark(1)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x1
|
|
; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
|
|
; SDAG-NEXT: ; wait_asyncmark(0)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x0
|
|
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
|
|
; SDAG-NEXT: s_wait_loadcnt 0x2
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v4, v11, v10
|
|
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
|
|
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
|
|
; SDAG-NEXT: v_add3_u32 v0, v4, v3, v0
|
|
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; SDAG-NEXT: v_add3_u32 v0, v0, v1, v2
|
|
; SDAG-NEXT: global_store_b32 v[6:7], v0, off
|
|
; SDAG-NEXT: s_set_pc_i64 s[30:31]
|
|
;
|
|
; GISEL-LABEL: interleaved_with_wave_barrier:
|
|
; GISEL: ; %bb.0: ; %entry
|
|
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GISEL-NEXT: s_wait_kmcnt 0x0
|
|
; GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
|
|
; GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
|
|
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x54, v0
|
|
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
|
|
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
|
|
; GISEL-NEXT: global_load_b32 v10, v[8:9], off offset:44
|
|
; GISEL-NEXT: global_load_b32 v11, v[0:1], off offset:4
|
|
; GISEL-NEXT: ; wave barrier
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
|
|
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
|
|
; GISEL-NEXT: ; wave barrier
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
|
|
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
|
|
; GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:8
|
|
; GISEL-NEXT: ; wave barrier
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
|
|
; GISEL-NEXT: ; wave barrier
|
|
; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: ; wait_asyncmark(1)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x1
|
|
; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
|
|
; GISEL-NEXT: ; wait_asyncmark(0)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x0
|
|
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
|
|
; GISEL-NEXT: s_wait_loadcnt 0x2
|
|
; GISEL-NEXT: v_add_nc_u32_e32 v6, v11, v10
|
|
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
|
|
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
|
|
; GISEL-NEXT: v_add3_u32 v0, v6, v3, v0
|
|
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GISEL-NEXT: v_add3_u32 v0, v0, v1, v2
|
|
; GISEL-NEXT: global_store_b32 v[4:5], v0, off
|
|
; GISEL-NEXT: s_set_pc_i64 s[30:31]
|
|
entry:
|
|
; First batch: global load, global load, async global-to-LDS
|
|
%bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11
|
|
%bar_v11 = load i32, ptr addrspace(1) %bar_gep11
|
|
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
|
|
%foo_v1 = load i32, ptr addrspace(1) %foo_gep1
|
|
%lds_gep21 = getelementptr i32, ptr addrspace(3) %lds, i32 21
|
|
%bar_gep21 = getelementptr i32, ptr addrspace(1) %foo, i32 21
|
|
call void @llvm.amdgcn.wave.barrier()
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
|
|
call void @llvm.amdgcn.wave.barrier()
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Second batch: global load, async global-to-LDS, global load
|
|
%foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
|
|
%foo_v2 = load i32, ptr addrspace(1) %foo_gep2
|
|
%bar_gep22 = getelementptr i32, ptr addrspace(1) %bar, i32 22
|
|
%lds_gep22 = getelementptr i32, ptr addrspace(3) %lds, i32 22
|
|
call void @llvm.amdgcn.wave.barrier()
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep22, ptr addrspace(3) %lds_gep22, i32 4, i32 u0x23)
|
|
call void @llvm.amdgcn.wave.barrier()
|
|
%bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
|
|
%bar_v12 = load i32, ptr addrspace(1) %bar_gep12
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Wait for first async mark and read from LDS
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 1)
|
|
%lds_val21 = load i32, ptr addrspace(3) %lds_gep21
|
|
|
|
; Wait for the next async mark.
|
|
; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 0)
|
|
%lds_val22 = load i32, ptr addrspace(3) %lds_gep22
|
|
%sum1 = add i32 %foo_v1, %bar_v11
|
|
%sum2 = add i32 %sum1, %lds_val21
|
|
%sum3 = add i32 %sum2, %foo_v2
|
|
; Finally a loadcnt(0) for %bar_v12, which was not included in the async mark that followed it.
|
|
%sum4 = add i32 %sum3, %bar_v12
|
|
%sum5 = add i32 %sum4, %lds_val22
|
|
store i32 %sum5, ptr addrspace(1) %out
|
|
|
|
ret void
|
|
}
|
|
|
|
; A perfect loop that is unlikely to exist in real life. It uses only async
|
|
; operations, and result in waits that exactly match the stream of
|
|
; those outstanding operations.
|
|
|
|
define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
|
|
; SDAG-LABEL: test_pipelined_loop:
|
|
; SDAG: ; %bb.0: ; %prolog
|
|
; SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
|
|
; SDAG-NEXT: v_nop
|
|
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
|
|
; SDAG-NEXT: s_clause 0x1
|
|
; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
|
|
; SDAG-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
|
|
; SDAG-NEXT: s_wait_kmcnt 0x0
|
|
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
|
|
; SDAG-NEXT: s_add_co_i32 s6, s2, 4
|
|
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; SDAG-NEXT: v_dual_mov_b32 v2, 4 :: v_dual_mov_b32 v3, s6
|
|
; SDAG-NEXT: s_mov_b32 s6, 2
|
|
; SDAG-NEXT: s_mov_b32 s7, s2
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
|
|
; SDAG-NEXT: v_mov_b32_e32 v1, 0
|
|
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: .LBB1_1: ; %loop_body
|
|
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
|
|
; SDAG-NEXT: s_add_co_i32 s6, s6, 1
|
|
; SDAG-NEXT: v_mov_b32_e32 v2, s8
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
|
|
; SDAG-NEXT: v_mov_b32_e32 v2, s7
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: ; wait_asyncmark(2)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x2
|
|
; SDAG-NEXT: s_add_co_i32 s7, s7, 4
|
|
; SDAG-NEXT: s_cmp_lt_i32 s6, s3
|
|
; SDAG-NEXT: ds_load_b32 v2, v2
|
|
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
|
|
; SDAG-NEXT: s_wait_dscnt 0x0
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v2
|
|
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
|
|
; SDAG-NEXT: ; %bb.2: ; %epilog
|
|
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
|
|
; SDAG-NEXT: ; wait_asyncmark(1)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x1
|
|
; SDAG-NEXT: s_add_co_i32 s0, s0, -8
|
|
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
|
|
; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
|
|
; SDAG-NEXT: ds_load_b32 v0, v0
|
|
; SDAG-NEXT: ; wait_asyncmark(0)
|
|
; SDAG-NEXT: s_wait_dscnt 0x0
|
|
; SDAG-NEXT: s_wait_asynccnt 0x0
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
|
|
; SDAG-NEXT: s_wait_kmcnt 0x0
|
|
; SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
|
|
; SDAG-NEXT: s_endpgm
|
|
;
|
|
; GISEL-LABEL: test_pipelined_loop:
|
|
; GISEL: ; %bb.0: ; %prolog
|
|
; GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
|
|
; GISEL-NEXT: v_nop
|
|
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
|
|
; GISEL-NEXT: s_clause 0x1
|
|
; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
|
|
; GISEL-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, 0
|
|
; GISEL-NEXT: s_mov_b32 s7, 2
|
|
; GISEL-NEXT: s_wait_kmcnt 0x0
|
|
; GISEL-NEXT: s_add_co_u32 s6, s2, 4
|
|
; GISEL-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, 4
|
|
; GISEL-NEXT: v_mov_b32_e32 v3, s6
|
|
; GISEL-NEXT: s_mov_b32 s6, 0
|
|
; GISEL-NEXT: s_mov_b32 s8, s2
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
|
|
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
|
|
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: .LBB1_1: ; %loop_body
|
|
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
|
|
; GISEL-NEXT: s_add_co_i32 s7, s7, 1
|
|
; GISEL-NEXT: v_mov_b32_e32 v1, s9
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
|
|
; GISEL-NEXT: v_mov_b32_e32 v1, s8
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: ; wait_asyncmark(2)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x2
|
|
; GISEL-NEXT: ds_load_b32 v1, v1
|
|
; GISEL-NEXT: s_wait_dscnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s9, v1
|
|
; GISEL-NEXT: s_add_co_i32 s6, s6, s9
|
|
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
|
|
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
|
|
; GISEL-NEXT: s_add_co_u32 s8, s8, 4
|
|
; GISEL-NEXT: s_cmp_lt_i32 s7, s3
|
|
; GISEL-NEXT: s_cbranch_scc1 .LBB1_1
|
|
; GISEL-NEXT: ; %bb.2: ; %epilog
|
|
; GISEL-NEXT: s_lshl_b32 s0, s3, 2
|
|
; GISEL-NEXT: ; wait_asyncmark(1)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x1
|
|
; GISEL-NEXT: s_add_co_u32 s0, s2, s0
|
|
; GISEL-NEXT: v_mov_b32_e32 v1, 0
|
|
; GISEL-NEXT: s_add_co_u32 s0, s0, -8
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, s0
|
|
; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
|
|
; GISEL-NEXT: ds_load_b32 v0, v0
|
|
; GISEL-NEXT: ; wait_asyncmark(0)
|
|
; GISEL-NEXT: s_wait_dscnt 0x0
|
|
; GISEL-NEXT: s_wait_asynccnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s2, v0
|
|
; GISEL-NEXT: s_add_co_i32 s2, s6, s2
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, s2
|
|
; GISEL-NEXT: s_wait_kmcnt 0x0
|
|
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
|
|
; GISEL-NEXT: s_endpgm
|
|
prolog:
|
|
; Load first iteration
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Load second iteration
|
|
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
|
|
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
br label %loop_body
|
|
|
|
loop_body:
|
|
%i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
|
|
%sum = phi i32 [ 0, %prolog ], [ %sum_i, %loop_body ]
|
|
|
|
; Load next iteration
|
|
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
|
|
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Wait for iteration i-2 and process
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 2)
|
|
%lds_idx = sub i32 %i, 2
|
|
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
|
|
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
|
|
|
|
%sum_i = add i32 %sum, %lds_val
|
|
|
|
%i.next = add i32 %i, 1
|
|
%cmp = icmp slt i32 %i.next, %n
|
|
br i1 %cmp, label %loop_body, label %epilog
|
|
|
|
epilog:
|
|
; Process remaining iterations
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 1)
|
|
%lds_n_2 = sub i32 %n, 2
|
|
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
|
|
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
|
|
%sum_e2 = add i32 %sum_i, %lds_val_n_2
|
|
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
|
|
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 0)
|
|
%lds_n_1 = sub i32 %n, 1
|
|
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
|
|
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
|
|
%sum_e1 = add i32 %sum_e2, %lds_val_n_1
|
|
store i32 %sum_e2, ptr addrspace(1) %bar
|
|
|
|
ret void
|
|
}
|
|
|
|
; Software pipelined loop with async global-to-LDS and global loads
|
|
|
|
define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
|
|
; SDAG-LABEL: test_pipelined_loop_with_global:
|
|
; SDAG: ; %bb.0: ; %prolog
|
|
; SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
|
|
; SDAG-NEXT: v_nop
|
|
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
|
|
; SDAG-NEXT: s_clause 0x1
|
|
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
|
|
; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
|
|
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 4
|
|
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
|
|
; SDAG-NEXT: s_wait_kmcnt 0x0
|
|
; SDAG-NEXT: s_add_co_i32 s7, s10, 4
|
|
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; SDAG-NEXT: v_dual_mov_b32 v1, s10 :: v_dual_mov_b32 v4, s7
|
|
; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0
|
|
; SDAG-NEXT: s_load_b32 s12, s[0:1], 0x0
|
|
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
|
|
; SDAG-NEXT: s_clause 0x2
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
|
|
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
|
|
; SDAG-NEXT: s_wait_xcnt 0x0
|
|
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
|
|
; SDAG-NEXT: s_mov_b32 s8, 2
|
|
; SDAG-NEXT: s_mov_b32 s9, s10
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: s_wait_kmcnt 0x0
|
|
; SDAG-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s12
|
|
; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
|
|
; SDAG-NEXT: s_wait_loadcnt 0x0
|
|
; SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
|
|
; SDAG-NEXT: .LBB2_1: ; %loop_body
|
|
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
|
|
; SDAG-NEXT: s_wait_loadcnt 0x0
|
|
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
|
|
; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
|
|
; SDAG-NEXT: v_mov_b32_e32 v8, v3
|
|
; SDAG-NEXT: s_clause 0x1
|
|
; SDAG-NEXT: global_load_b32 v3, v0, s[4:5]
|
|
; SDAG-NEXT: global_load_b32 v4, v0, s[0:1]
|
|
; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
|
|
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
|
|
; SDAG-NEXT: v_mov_b32_e32 v9, s9
|
|
; SDAG-NEXT: ; asyncmark
|
|
; SDAG-NEXT: ; wait_asyncmark(2)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x2
|
|
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
|
|
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
|
|
; SDAG-NEXT: ds_load_b32 v9, v9
|
|
; SDAG-NEXT: v_mov_b32_e32 v5, v1
|
|
; SDAG-NEXT: s_cmp_lt_i32 s8, s11
|
|
; SDAG-NEXT: s_wait_xcnt 0x0
|
|
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
|
|
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[4:5], 4
|
|
; SDAG-NEXT: s_wait_dscnt 0x0
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v9, v10, v9
|
|
; SDAG-NEXT: global_store_b32 v0, v9, s[6:7]
|
|
; SDAG-NEXT: s_wait_xcnt 0x0
|
|
; SDAG-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4
|
|
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
|
|
; SDAG-NEXT: ; %bb.2: ; %epilog
|
|
; SDAG-NEXT: s_add_co_i32 s0, s11, -2
|
|
; SDAG-NEXT: ; wait_asyncmark(1)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x1
|
|
; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10
|
|
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; SDAG-NEXT: v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1
|
|
; SDAG-NEXT: ds_load_b32 v1, v0
|
|
; SDAG-NEXT: s_wait_dscnt 0x0
|
|
; SDAG-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v2, v1
|
|
; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
|
|
; SDAG-NEXT: ; wait_asyncmark(0)
|
|
; SDAG-NEXT: s_wait_asynccnt 0x0
|
|
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
|
|
; SDAG-NEXT: s_wait_loadcnt 0x0
|
|
; SDAG-NEXT: s_wait_xcnt 0x0
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v1, v3, v4
|
|
; SDAG-NEXT: s_wait_dscnt 0x0
|
|
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
|
|
; SDAG-NEXT: global_store_b32 v5, v0, s[2:3] offset:4 scale_offset
|
|
; SDAG-NEXT: s_endpgm
|
|
;
|
|
; GISEL-LABEL: test_pipelined_loop_with_global:
|
|
; GISEL: ; %bb.0: ; %prolog
|
|
; GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
|
|
; GISEL-NEXT: v_nop
|
|
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
|
|
; GISEL-NEXT: s_clause 0x1
|
|
; GISEL-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
|
|
; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, 0
|
|
; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
|
|
; GISEL-NEXT: s_mov_b32 s13, 2
|
|
; GISEL-NEXT: s_wait_kmcnt 0x0
|
|
; GISEL-NEXT: s_load_b32 s12, s[8:9], 0x0
|
|
; GISEL-NEXT: s_load_b32 s17, s[0:1], 0x0
|
|
; GISEL-NEXT: v_mov_b32_e32 v1, s10
|
|
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s6
|
|
; GISEL-NEXT: s_mov_b32 s14, s10
|
|
; GISEL-NEXT: s_clause 0x2
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
|
|
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
|
|
; GISEL-NEXT: s_wait_xcnt 0x0
|
|
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
|
|
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
|
|
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
|
|
; GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
|
|
; GISEL-NEXT: s_add_co_ci_u32 s7, s9, 0
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: s_wait_loadcnt 0x1
|
|
; GISEL-NEXT: v_readfirstlane_b32 s15, v1
|
|
; GISEL-NEXT: s_wait_loadcnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s16, v2
|
|
; GISEL-NEXT: s_mov_b32 s9, s15
|
|
; GISEL-NEXT: s_mov_b32 s19, s16
|
|
; GISEL-NEXT: .LBB2_1: ; %loop_body
|
|
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GISEL-NEXT: s_add_co_u32 s8, s14, 8
|
|
; GISEL-NEXT: s_clause 0x1
|
|
; GISEL-NEXT: global_load_b32 v1, v0, s[6:7]
|
|
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1]
|
|
; GISEL-NEXT: v_mov_b32_e32 v3, s8
|
|
; GISEL-NEXT: s_mov_b32 s18, s9
|
|
; GISEL-NEXT: s_wait_kmcnt 0x0
|
|
; GISEL-NEXT: s_add_co_i32 s9, s12, s17
|
|
; GISEL-NEXT: s_add_co_i32 s13, s13, 1
|
|
; GISEL-NEXT: s_mov_b32 s8, s19
|
|
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[6:7] offset:4 nv
|
|
; GISEL-NEXT: v_mov_b32_e32 v3, s14
|
|
; GISEL-NEXT: ; asyncmark
|
|
; GISEL-NEXT: ; wait_asyncmark(2)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x2
|
|
; GISEL-NEXT: s_mov_b32 s12, s15
|
|
; GISEL-NEXT: ds_load_b32 v3, v3
|
|
; GISEL-NEXT: s_wait_dscnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s17, v3
|
|
; GISEL-NEXT: s_add_co_i32 s9, s9, s17
|
|
; GISEL-NEXT: s_wait_xcnt 0x0
|
|
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
|
|
; GISEL-NEXT: v_mov_b32_e32 v3, s9
|
|
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
|
|
; GISEL-NEXT: s_add_co_u32 s6, s6, 4
|
|
; GISEL-NEXT: s_add_co_ci_u32 s7, s7, 0
|
|
; GISEL-NEXT: s_mov_b32 s17, s16
|
|
; GISEL-NEXT: global_store_b32 v0, v3, s[4:5]
|
|
; GISEL-NEXT: s_wait_xcnt 0x0
|
|
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
|
|
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
|
|
; GISEL-NEXT: s_add_co_u32 s14, s14, 4
|
|
; GISEL-NEXT: s_cmp_lt_i32 s13, s11
|
|
; GISEL-NEXT: s_wait_loadcnt 0x1
|
|
; GISEL-NEXT: v_readfirstlane_b32 s9, v1
|
|
; GISEL-NEXT: s_wait_loadcnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s19, v2
|
|
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
|
|
; GISEL-NEXT: ; %bb.2: ; %epilog
|
|
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
|
|
; GISEL-NEXT: ; wait_asyncmark(1)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x1
|
|
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
|
|
; GISEL-NEXT: s_add_co_i32 s4, s18, s8
|
|
; GISEL-NEXT: s_add_co_u32 s1, s10, s1
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, s1
|
|
; GISEL-NEXT: ds_load_b32 v1, v0
|
|
; GISEL-NEXT: s_wait_dscnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s1, v1
|
|
; GISEL-NEXT: v_mov_b32_e32 v1, s0
|
|
; GISEL-NEXT: s_add_co_i32 s1, s4, s1
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_mov_b32_e32 v2, s1
|
|
; GISEL-NEXT: s_add_co_i32 s1, s9, s19
|
|
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
|
|
; GISEL-NEXT: ; wait_asyncmark(0)
|
|
; GISEL-NEXT: s_wait_asynccnt 0x0
|
|
; GISEL-NEXT: ds_load_b32 v0, v0 offset:4
|
|
; GISEL-NEXT: s_wait_dscnt 0x0
|
|
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
|
|
; GISEL-NEXT: s_add_co_i32 s0, s1, s0
|
|
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GISEL-NEXT: v_mov_b32_e32 v0, s0
|
|
; GISEL-NEXT: global_store_b32 v1, v0, s[2:3] offset:4 scale_offset
|
|
; GISEL-NEXT: s_endpgm
|
|
prolog:
|
|
; Load first iteration
|
|
%v0 = load i32, ptr addrspace(1) %foo
|
|
%g0 = load i32, ptr addrspace(1) %bar
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Load second iteration
|
|
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
|
|
%v1 = load i32, ptr addrspace(1) %foo_gep1
|
|
%bar_gep1 = getelementptr i32, ptr addrspace(1) %bar, i32 1
|
|
%g1 = load i32, ptr addrspace(1) %bar_gep1
|
|
|
|
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
br label %loop_body
|
|
|
|
loop_body:
|
|
%i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
|
|
%prev_v = phi i32 [ %v0, %prolog ], [ %v1, %loop_body ]
|
|
%prev_g = phi i32 [ %g0, %prolog ], [ %g1, %loop_body ]
|
|
%v1_phi = phi i32 [ %v1, %prolog ], [ %cur_v, %loop_body ]
|
|
%g1_phi = phi i32 [ %g1, %prolog ], [ %cur_g, %loop_body ]
|
|
|
|
; Load next iteration
|
|
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
|
|
%cur_v = load i32, ptr addrspace(1) %foo_gep_cur
|
|
%bar_gep_cur = getelementptr i32, ptr addrspace(1) %bar, i32 %i
|
|
%cur_g = load i32, ptr addrspace(1) %bar_gep_cur
|
|
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
|
|
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
|
|
call void @llvm.amdgcn.asyncmark()
|
|
|
|
; Wait for iteration i-2 and process
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 2)
|
|
%lds_idx = sub i32 %i, 2
|
|
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
|
|
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
|
|
|
|
%sum1 = add i32 %prev_v, %prev_g
|
|
%sum2 = add i32 %sum1, %lds_val
|
|
%out_gep = getelementptr i32, ptr addrspace(1) %out, i32 %lds_idx
|
|
store i32 %sum2, ptr addrspace(1) %out_gep
|
|
|
|
%i.next = add i32 %i, 1
|
|
%cmp = icmp slt i32 %i.next, %n
|
|
br i1 %cmp, label %loop_body, label %epilog
|
|
|
|
epilog:
|
|
; Process remaining iterations
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 1)
|
|
%lds_n_2 = sub i32 %n, 2
|
|
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
|
|
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
|
|
%sum_e1 = add i32 %v1_phi, %g1_phi
|
|
%sum_e2 = add i32 %sum_e1, %lds_val_n_2
|
|
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
|
|
store i32 %sum_e2, ptr addrspace(1) %out_gep_e1
|
|
|
|
call void @llvm.amdgcn.wait.asyncmark(i16 0)
|
|
%lds_n_1 = sub i32 %n, 1
|
|
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
|
|
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
|
|
%sum_e3 = add i32 %cur_v, %cur_g
|
|
%sum_e4 = add i32 %sum_e3, %lds_val_n_1
|
|
%out_gep_e2 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_1
|
|
store i32 %sum_e4, ptr addrspace(1) %out_gep_e2
|
|
|
|
ret void
|
|
}
|