; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -amdgpu-sched-strategy=coexec --enable-post-misched=0 --verify-misched < %s | FileCheck -check-prefix=COEXEC %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 { ; COEXEC-LABEL: ds_wmma: ; COEXEC: ; %bb.0: ; %entry ; COEXEC-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; COEXEC-NEXT: v_mov_b32_e32 v0, 0 ; COEXEC-NEXT: s_clause 0x1 ; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv ; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv ; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0 ; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0 ; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0 ; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0 ; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0 ; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0 ; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0 ; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0 ; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0 ; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0 ; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0 ; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0 ; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0 ; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0 ; COEXEC-NEXT: s_wait_kmcnt 0x0 ; COEXEC-NEXT: s_bitcmp1_b32 s0, 0 ; COEXEC-NEXT: v_mov_b32_e32 v29, v0 ; COEXEC-NEXT: s_cselect_b32 s0, -1, 0 ; COEXEC-NEXT: v_mov_b32_e32 v30, v0 ; COEXEC-NEXT: s_xor_b32 s0, s0, -1 ; COEXEC-NEXT: v_mov_b32_e32 v31, v0 ; COEXEC-NEXT: v_cndmask_b32_e64 v32, 0, 1, s0 ; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v32 ; COEXEC-NEXT: .LBB0_1: ; %loop ; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1 ; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0 ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_mov_b32_e32 v88, s2 ; COEXEC-NEXT: s_add_co_i32 s2, s2, s1 ; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v88 offset:192 ; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v88 ; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v88 offset:64 ; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v88 offset:128 ; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v88 offset:448 ; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v88 offset:384 ; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v88 offset:256 ; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v88 offset:320 ; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v88 offset:704 ; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v88 offset:640 ; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v88 offset:576 ; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v88 offset:512 ; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v88 offset:960 ; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v88 offset:896 ; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v88 offset:832 ; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v88 offset:768 ; COEXEC-NEXT: s_wait_dscnt 0xc ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31] ; COEXEC-NEXT: s_wait_dscnt 0x8 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23] ; COEXEC-NEXT: s_wait_dscnt 0x4 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15] ; COEXEC-NEXT: s_wait_dscnt 0x0 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7] ; COEXEC-NEXT: s_cbranch_vccnz .LBB0_1 ; COEXEC-NEXT: ; %bb.2: ; %end ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_mov_b32_e32 v32, 0 ; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv ; COEXEC-NEXT: s_wait_kmcnt 0x0 ; COEXEC-NEXT: s_clause 0x7 ; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16 ; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1] ; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144 ; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128 ; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272 ; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256 ; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400 ; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384 ; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; COEXEC-NEXT: s_endpgm ; ; GCN-LABEL: ds_wmma: ; GCN: ; %bb.0: ; %entry ; GCN-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE ; GCN-NEXT: v_nop ; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; GCN-NEXT: s_clause 0x1 ; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv ; GCN-NEXT: s_load_b32 s2, s[4:5], 0x0 nv ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0 ; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0 ; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0 ; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0 ; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0 ; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0 ; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0 ; GCN-NEXT: s_wait_kmcnt 0x0 ; GCN-NEXT: s_bitcmp1_b32 s0, 0 ; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0 ; GCN-NEXT: s_xor_b32 s0, s0, -1 ; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0 ; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0 ; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0 ; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0 ; GCN-NEXT: v_mov_b32_e32 v26, v0 ; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24 ; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0 ; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0 ; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0 ; GCN-NEXT: .LBB0_1: ; %loop ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN-NEXT: v_nop ; GCN-NEXT: v_nop ; GCN-NEXT: v_nop ; GCN-NEXT: v_nop ; GCN-NEXT: v_mov_b32_e32 v92, s2 ; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0 ; GCN-NEXT: s_add_co_i32 s2, s2, s1 ; GCN-NEXT: ds_load_tr16_b128 v[32:35], v92 ; GCN-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:64 ; GCN-NEXT: ds_load_tr16_b128 v[40:43], v92 offset:128 ; GCN-NEXT: ds_load_tr16_b128 v[44:47], v92 offset:192 ; GCN-NEXT: ds_load_tr16_b128 v[48:51], v92 offset:256 ; GCN-NEXT: ds_load_tr16_b128 v[52:55], v92 offset:320 ; GCN-NEXT: ds_load_tr16_b128 v[56:59], v92 offset:384 ; GCN-NEXT: ds_load_tr16_b128 v[60:63], v92 offset:448 ; GCN-NEXT: ds_load_tr16_b128 v[64:67], v92 offset:512 ; GCN-NEXT: ds_load_tr16_b128 v[68:71], v92 offset:576 ; GCN-NEXT: ds_load_tr16_b128 v[72:75], v92 offset:640 ; GCN-NEXT: ds_load_tr16_b128 v[76:79], v92 offset:704 ; GCN-NEXT: ds_load_tr16_b128 v[80:83], v92 offset:768 ; GCN-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:832 ; GCN-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:896 ; GCN-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:960 ; GCN-NEXT: s_wait_dscnt 0xc ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: s_wait_dscnt 0x8 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: s_wait_dscnt 0x4 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: s_cbranch_vccnz .LBB0_1 ; GCN-NEXT: ; %bb.2: ; %end ; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv ; GCN-NEXT: v_nop ; GCN-NEXT: v_mov_b32_e32 v32, 0 ; GCN-NEXT: s_wait_kmcnt 0x0 ; GCN-NEXT: s_clause 0x7 ; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16 ; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1] ; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144 ; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128 ; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272 ; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256 ; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400 ; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384 ; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GCN-NEXT: s_endpgm entry: br label %loop loop: %baseOff = phi i32 [ 0, %entry ], [ %newBaseOff, %loop ] %wvec0 = phi <8 x float> [ , %entry ], [ %wmma01, %loop ] %wvec1 = phi <8 x float> [ , %entry ], [ %wmma11, %loop ] %wvec2 = phi <8 x float> [ , %entry ], [ %wmma21, %loop ] %wvec3 = phi <8 x float> [ , %entry ], [ %wmma31, %loop ] %p0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base, i32 %baseOff %p1 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 64 %p2 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 128 %p3 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 192 %p4 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 256 %p5 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 320 %p6 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 384 %p7 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 448 %p8 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 512 %p9 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 576 %p10 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 640 %p11 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 704 %p12 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 768 %p13 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 832 %p14 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 896 %p15 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 960 %l0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %p0) %l1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p1) %l2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p2) %l3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p3) %l4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p4) %l5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p5) %l6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p6) %l7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p7) %l8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p8) %l9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p9) %l10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p10) %l11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p11) %l12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p12) %l13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p13) %l14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p14) %l15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p15) %vec0 = shufflevector <8 x half> %l0, <8 x half> %l1, <16 x i32> %vec1 = shufflevector <8 x half> %l2, <8 x half> %l3, <16 x i32> %vec2 = shufflevector <8 x half> %l4, <8 x half> %l5, <16 x i32> %vec3 = shufflevector <8 x half> %l6, <8 x half> %l7, <16 x i32> %vec4 = shufflevector <8 x half> %l8, <8 x half> %l9, <16 x i32> %vec5 = shufflevector <8 x half> %l10, <8 x half> %l11, <16 x i32> %vec6 = shufflevector <8 x half> %l12, <8 x half> %l13, <16 x i32> %vec7 = shufflevector <8 x half> %l14, <8 x half> %l15, <16 x i32> %wmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %vec1, i16 0, <8 x float> %wvec0, i1 false, i1 false) %wmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %vec1, i16 0, <8 x float> %wmma00, i1 false, i1 false) %wmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %vec3, i16 0, <8 x float> %wvec1, i1 false, i1 false) %wmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %vec3, i16 0, <8 x float> %wmma10, i1 false, i1 false) %wmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %vec5, i16 0, <8 x float> %wvec2, i1 false, i1 false) %wmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %vec5, i16 0, <8 x float> %wmma20, i1 false, i1 false) %wmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %vec7, i16 0, <8 x float> %wvec3, i1 false, i1 false) %wmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %vec7, i16 0, <8 x float> %wmma30, i1 false, i1 false) %newBaseOff = or disjoint i32 %baseOff, %delta br i1 %br0, label %loop, label %end end: %out1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 128 %out2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 256 %out3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 384 store <8 x float> %wmma01, ptr addrspace(1) %out, align 16 store <8 x float> %wmma11, ptr addrspace(1) %out1, align 16 store <8 x float> %wmma21, ptr addrspace(1) %out2, align 16 store <8 x float> %wmma31, ptr addrspace(1) %out3, align 16 ret void } define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace(3) %base1, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 { ; COEXEC-LABEL: ds_wmma_permute: ; COEXEC: ; %bb.0: ; %entry ; COEXEC-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; COEXEC-NEXT: s_mov_b32 s6, 0 ; COEXEC-NEXT: s_clause 0x1 ; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv ; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv ; COEXEC-NEXT: v_mov_b32_e32 v0, 0 ; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0 ; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0 ; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0 ; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0 ; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0 ; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0 ; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0 ; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0 ; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0 ; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0 ; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0 ; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0 ; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0 ; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0 ; COEXEC-NEXT: s_wait_kmcnt 0x0 ; COEXEC-NEXT: s_bitcmp1_b32 s0, 0 ; COEXEC-NEXT: v_mov_b32_e32 v29, v0 ; COEXEC-NEXT: s_cselect_b32 s0, -1, 0 ; COEXEC-NEXT: v_mov_b32_e32 v30, v0 ; COEXEC-NEXT: s_xor_b32 s0, s0, -1 ; COEXEC-NEXT: v_mov_b32_e32 v31, v0 ; COEXEC-NEXT: v_cndmask_b32_e64 v32, 0, 1, s0 ; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v32 ; COEXEC-NEXT: .LBB1_1: ; %loop ; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1 ; COEXEC-NEXT: s_add_co_i32 s7, s2, s6 ; COEXEC-NEXT: s_add_co_i32 s8, s3, s6 ; COEXEC-NEXT: s_add_co_i32 s6, s6, s1 ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_nop ; COEXEC-NEXT: v_mov_b32_e32 v124, s7 ; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0 ; COEXEC-NEXT: v_mov_b32_e32 v156, s8 ; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v124 ; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v124 offset:64 ; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v156 ; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v156 offset:64 ; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v124 offset:320 ; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v124 offset:256 ; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v156 offset:256 ; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v156 offset:320 ; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v124 offset:576 ; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v124 offset:512 ; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v156 offset:576 ; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v156 offset:512 ; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v124 offset:832 ; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v124 offset:768 ; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v156 offset:832 ; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v156 offset:768 ; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128 ; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192 ; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384 ; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448 ; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640 ; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704 ; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896 ; COEXEC-NEXT: s_wait_dscnt 0x13 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960 ; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128 ; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192 ; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384 ; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448 ; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640 ; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704 ; COEXEC-NEXT: s_wait_dscnt 0x16 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896 ; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960 ; COEXEC-NEXT: s_wait_dscnt 0x14 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; COEXEC-NEXT: s_wait_dscnt 0x10 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; COEXEC-NEXT: s_wait_dscnt 0x6 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31] ; COEXEC-NEXT: s_wait_dscnt 0x4 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23] ; COEXEC-NEXT: s_wait_dscnt 0x2 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15] ; COEXEC-NEXT: s_wait_dscnt 0x0 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15] ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7] ; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1 ; COEXEC-NEXT: ; %bb.2: ; %end ; COEXEC-NEXT: v_mov_b32_e32 v32, 0 ; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv ; COEXEC-NEXT: s_wait_kmcnt 0x0 ; COEXEC-NEXT: s_clause 0x7 ; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16 ; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1] ; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144 ; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128 ; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272 ; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256 ; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400 ; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384 ; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; COEXEC-NEXT: s_endpgm ; ; GCN-LABEL: ds_wmma_permute: ; GCN: ; %bb.0: ; %entry ; GCN-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE ; GCN-NEXT: v_nop ; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; GCN-NEXT: s_clause 0x1 ; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv ; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_mov_b32 s6, 0 ; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0 ; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0 ; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0 ; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0 ; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0 ; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0 ; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0 ; GCN-NEXT: s_wait_kmcnt 0x0 ; GCN-NEXT: s_bitcmp1_b32 s0, 0 ; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0 ; GCN-NEXT: s_xor_b32 s0, s0, -1 ; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0 ; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0 ; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0 ; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0 ; GCN-NEXT: v_mov_b32_e32 v26, v0 ; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24 ; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0 ; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0 ; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0 ; GCN-NEXT: .LBB1_1: ; %loop ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 ; GCN-NEXT: s_add_co_i32 s7, s2, s6 ; GCN-NEXT: s_add_co_i32 s8, s3, s6 ; GCN-NEXT: v_dual_mov_b32 v96, s7 :: v_dual_mov_b32 v97, s8 ; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0 ; GCN-NEXT: s_add_co_i32 s6, s6, s1 ; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96 ; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:64 ; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97 ; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:64 ; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:256 ; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:320 ; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:256 ; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:320 ; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:512 ; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:576 ; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:512 ; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:576 ; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:768 ; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:832 ; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:768 ; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:832 ; GCN-NEXT: s_wait_dscnt 0xc ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: s_wait_dscnt 0x8 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: s_wait_dscnt 0x4 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96 offset:128 ; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:192 ; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97 offset:128 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:192 ; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:384 ; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:448 ; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:384 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:448 ; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:640 ; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:704 ; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:640 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:704 ; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:896 ; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:960 ; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:896 ; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:960 ; GCN-NEXT: s_wait_dscnt 0xc ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: s_wait_dscnt 0x8 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: s_wait_dscnt 0x4 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7] ; GCN-NEXT: s_cbranch_vccnz .LBB1_1 ; GCN-NEXT: ; %bb.2: ; %end ; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv ; GCN-NEXT: v_nop ; GCN-NEXT: v_mov_b32_e32 v32, 0 ; GCN-NEXT: s_wait_kmcnt 0x0 ; GCN-NEXT: s_clause 0x7 ; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16 ; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1] ; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144 ; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128 ; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272 ; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256 ; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400 ; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384 ; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GCN-NEXT: s_endpgm entry: br label %loop loop: %baseOff = phi i32 [ 0, %entry ], [ %newBaseOff, %loop ] %wvec0 = phi <8 x float> [ , %entry ], [ %bwmma01, %loop ] %wvec1 = phi <8 x float> [ , %entry ], [ %bwmma11, %loop ] %wvec2 = phi <8 x float> [ , %entry ], [ %bwmma21, %loop ] %wvec3 = phi <8 x float> [ , %entry ], [ %bwmma31, %loop ] %p0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base, i32 %baseOff %p1 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 64 %p2 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 128 %p3 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 192 %p4 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 256 %p5 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 320 %p6 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 384 %p7 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 448 %p8 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 512 %p9 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 576 %p10 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 640 %p11 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 704 %p12 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 768 %p13 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 832 %p14 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 896 %p15 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 960 %bp0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base1, i32 %baseOff %bp1 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 64 %bp2 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 128 %bp3 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 192 %bp4 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 256 %bp5 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 320 %bp6 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 384 %bp7 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 448 %bp8 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 512 %bp9 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 576 %bp10 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 640 %bp11 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 704 %bp12 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 768 %bp13 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 832 %bp14 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 896 %bp15 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 960 %l0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %p0) %l1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p1) %l2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p2) %l3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p3) %l4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p4) %l5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p5) %l6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p6) %l7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p7) %l8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p8) %l9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p9) %l10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p10) %l11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p11) %l12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p12) %l13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p13) %l14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p14) %l15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p15) %bl0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %bp0) %bl1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp1) %bl2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp2) %bl3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp3) %bl4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp4) %bl5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp5) %bl6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp6) %bl7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp7) %bl8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp8) %bl9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp9) %bl10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp10) %bl11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp11) %bl12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp12) %bl13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp13) %bl14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp14) %bl15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp15) %vec0 = shufflevector <8 x half> %l0, <8 x half> %l1, <16 x i32> %vec1 = shufflevector <8 x half> %l2, <8 x half> %l3, <16 x i32> %vec2 = shufflevector <8 x half> %l4, <8 x half> %l5, <16 x i32> %vec3 = shufflevector <8 x half> %l6, <8 x half> %l7, <16 x i32> %vec4 = shufflevector <8 x half> %l8, <8 x half> %l9, <16 x i32> %vec5 = shufflevector <8 x half> %l10, <8 x half> %l11, <16 x i32> %vec6 = shufflevector <8 x half> %l12, <8 x half> %l13, <16 x i32> %vec7 = shufflevector <8 x half> %l14, <8 x half> %l15, <16 x i32> %bvec0 = shufflevector <8 x half> %bl0, <8 x half> %bl1, <16 x i32> %bvec1 = shufflevector <8 x half> %bl2, <8 x half> %bl3, <16 x i32> %bvec2 = shufflevector <8 x half> %bl4, <8 x half> %bl5, <16 x i32> %bvec3 = shufflevector <8 x half> %bl6, <8 x half> %bl7, <16 x i32> %bvec4 = shufflevector <8 x half> %bl8, <8 x half> %bl9, <16 x i32> %bvec5 = shufflevector <8 x half> %bl10, <8 x half> %bl11, <16 x i32> %bvec6 = shufflevector <8 x half> %bl12, <8 x half> %bl13, <16 x i32> %bvec7 = shufflevector <8 x half> %bl14, <8 x half> %bl15, <16 x i32> %wmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %bvec0, i16 0, <8 x float> %wvec0, i1 false, i1 false) %bwmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %bvec0, i16 0, <8 x float> %wmma00, i1 false, i1 false) %wmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec1, i1 false, <16 x half> %bvec1, i16 0, <8 x float> %bwmma00, i1 false, i1 false) %bwmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec1, i1 false, <16 x half> %bvec1, i16 0, <8 x float> %wmma01, i1 false, i1 false) %wmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %bvec2, i16 0, <8 x float> %wvec1, i1 false, i1 false) %bwmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %bvec2, i16 0, <8 x float> %wmma10, i1 false, i1 false) %wmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec3, i1 false, <16 x half> %bvec3, i16 0, <8 x float> %bwmma10, i1 false, i1 false) %bwmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec3, i1 false, <16 x half> %bvec3, i16 0, <8 x float> %wmma11, i1 false, i1 false) %wmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %bvec4, i16 0, <8 x float> %wvec2, i1 false, i1 false) %bwmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %bvec4, i16 0, <8 x float> %wmma20, i1 false, i1 false) %wmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec5, i1 false, <16 x half> %bvec5, i16 0, <8 x float> %bwmma20, i1 false, i1 false) %bwmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec5, i1 false, <16 x half> %bvec5, i16 0, <8 x float> %wmma21, i1 false, i1 false) %wmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %bvec6, i16 0, <8 x float> %wvec3, i1 false, i1 false) %bwmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %bvec6, i16 0, <8 x float> %wmma30, i1 false, i1 false) %wmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec7, i1 false, <16 x half> %bvec7, i16 0, <8 x float> %bwmma30, i1 false, i1 false) %bwmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec7, i1 false, <16 x half> %bvec7, i16 0, <8 x float> %wmma31, i1 false, i1 false) %newBaseOff = or disjoint i32 %baseOff, %delta br i1 %br0, label %loop, label %end end: %out1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 128 %out2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 256 %out3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 384 store <8 x float> %bwmma01, ptr addrspace(1) %out, align 16 store <8 x float> %bwmma11, ptr addrspace(1) %out1, align 16 store <8 x float> %bwmma21, ptr addrspace(1) %out2, align 16 store <8 x float> %bwmma31, ptr addrspace(1) %out3, align 16 ret void } attributes #0 = { "amdgpu-flat-work-group-size"="32,32" "amdgpu-waves-per-eu"="1,1" }