; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s ; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s ; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefix=GFX90A %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GFX90A-VGPR %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16>, <2 x i16>, <32 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16>, <2 x i16>, <16 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.4x4x2bf16(<2 x i16>, <2 x i16>, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16>, <2 x i16>, <16 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16>, <2 x i16>, <4 x float>, i32, i32, i32) declare i32 @llvm.amdgcn.workitem.id.x() define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-LABEL: test_mfma_f32_32x32x2bf16: ; GFX908-SDAG: ; %bb.0: ; %bb ; GFX908-SDAG-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s16 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s18 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s21 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s22 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s23 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s24 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s25 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s26 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s27 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s28 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s29 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s30 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s31 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a16, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s3 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a17, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a18, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a19, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s4 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s5 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s6 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a20, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a21, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a22, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s7 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s8 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s9 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, s19 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a23, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a24, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a25, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s20 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v3 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a26, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a27, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a28, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s15 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v5 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a29, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a30, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a31, v2 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 ; GFX908-SDAG-NEXT: s_nop 15 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a27 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] ; GFX908-SDAG-NEXT: s_nop 0 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 ; GFX908-SDAG-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2bf16: ; GFX908-GISEL: ; %bb.0: ; %bb ; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 ; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-GISEL-NEXT: s_nop 1 ; GFX908-GISEL-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3 ; GFX908-GISEL-NEXT: s_nop 15 ; GFX908-GISEL-NEXT: s_nop 1 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 ; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 ; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x2bf16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v1, 1 ; GFX90A-NEXT: v_mov_b32_e32 v2, 2 ; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s16 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s17 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s18 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s19 ; GFX90A-NEXT: v_accvgpr_write_b32 a4, s20 ; GFX90A-NEXT: v_accvgpr_write_b32 a5, s21 ; GFX90A-NEXT: v_accvgpr_write_b32 a6, s22 ; GFX90A-NEXT: v_accvgpr_write_b32 a7, s23 ; GFX90A-NEXT: v_accvgpr_write_b32 a8, s24 ; GFX90A-NEXT: v_accvgpr_write_b32 a9, s25 ; GFX90A-NEXT: v_accvgpr_write_b32 a10, s26 ; GFX90A-NEXT: v_accvgpr_write_b32 a11, s27 ; GFX90A-NEXT: v_accvgpr_write_b32 a12, s28 ; GFX90A-NEXT: v_accvgpr_write_b32 a13, s29 ; GFX90A-NEXT: v_accvgpr_write_b32 a14, s30 ; GFX90A-NEXT: v_accvgpr_write_b32 a15, s31 ; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0 ; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2 ; GFX90A-NEXT: v_accvgpr_write_b32 a19, s3 ; GFX90A-NEXT: v_accvgpr_write_b32 a20, s4 ; GFX90A-NEXT: v_accvgpr_write_b32 a21, s5 ; GFX90A-NEXT: v_accvgpr_write_b32 a22, s6 ; GFX90A-NEXT: v_accvgpr_write_b32 a23, s7 ; GFX90A-NEXT: v_accvgpr_write_b32 a24, s8 ; GFX90A-NEXT: v_accvgpr_write_b32 a25, s9 ; GFX90A-NEXT: v_accvgpr_write_b32 a26, s10 ; GFX90A-NEXT: v_accvgpr_write_b32 a27, s11 ; GFX90A-NEXT: v_accvgpr_write_b32 a28, s12 ; GFX90A-NEXT: v_accvgpr_write_b32 a29, s13 ; GFX90A-NEXT: v_accvgpr_write_b32 a30, s14 ; GFX90A-NEXT: v_accvgpr_write_b32 a31, s15 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v1, v2, a[0:31] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 15 ; GFX90A-NEXT: s_nop 2 ; GFX90A-NEXT: global_store_dwordx4 v0, a[24:27], s[34:35] offset:96 ; GFX90A-NEXT: global_store_dwordx4 v0, a[28:31], s[34:35] offset:112 ; GFX90A-NEXT: global_store_dwordx4 v0, a[16:19], s[34:35] offset:64 ; GFX90A-NEXT: global_store_dwordx4 v0, a[20:23], s[34:35] offset:80 ; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[34:35] offset:32 ; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[34:35] offset:48 ; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[34:35] ; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[34:35] offset:16 ; GFX90A-NEXT: s_endpgm ; ; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x2bf16: ; GFX90A-VGPR: ; %bb.0: ; %bb ; GFX90A-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v33, 1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v34, 2 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v32, 0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, s16 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, s17 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, s18 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, s19 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, s20 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, s21 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, s22 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, s23 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, s24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v9, s25 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s26 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s27 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s28 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s29 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v14, s30 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v15, s31 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, s0 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, s1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, s2 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, s3 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v20, s4 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v21, s5 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v22, s6 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v23, s7 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v24, s8 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v25, s9 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v26, s10 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v27, s11 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v28, s12 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v29, s13 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v30, s14 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v31, s15 ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x2bf16 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3 ; GFX90A-VGPR-NEXT: s_nop 15 ; GFX90A-VGPR-NEXT: s_nop 2 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] ; GFX90A-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 ; GFX90A-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> %b = bitcast i32 2 to <2 x i16> %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16> %a, <2 x i16> %b, <32 x float> %in.1, i32 1, i32 2, i32 3) store <32 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-LABEL: test_mfma_f32_16x16x2bf16: ; GFX908-SDAG: ; %bb.0: ; %bb ; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v12, 0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s0 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s3 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s6 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s9 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s12 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s15 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX908-SDAG-NEXT: s_nop 9 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4 ; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48 ; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32 ; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] ; GFX908-SDAG-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_16x16x2bf16: ; GFX908-GISEL: ; %bb.0: ; %bb ; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-GISEL-NEXT: s_nop 1 ; GFX908-GISEL-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX908-GISEL-NEXT: s_nop 9 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_16x16x2bf16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v0, 1 ; GFX90A-NEXT: v_mov_b32_e32 v1, 2 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_nop 9 ; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX90A-NEXT: s_endpgm ; ; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x2bf16: ; GFX90A-VGPR: ; %bb.0: ; %bb ; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 2 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x2bf16 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 0 ; GFX90A-VGPR-NEXT: s_nop 9 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX90A-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> %b = bitcast i32 2 to <2 x i16> %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16> %a, <2 x i16> %b, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_4x4x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-LABEL: test_mfma_f32_4x4x2bf16: ; GFX908: ; %bb.0: ; %bb ; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX908-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-NEXT: v_mov_b32_e32 v4, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: v_mov_b32_e32 v5, s0 ; GFX908-NEXT: v_mov_b32_e32 v2, s1 ; GFX908-NEXT: v_mov_b32_e32 v3, s2 ; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 ; GFX908-NEXT: v_mov_b32_e32 v5, s3 ; GFX908-NEXT: v_accvgpr_write_b32 a1, v2 ; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 ; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 ; GFX908-NEXT: s_nop 0 ; GFX908-NEXT: v_mfma_f32_4x4x2bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 ; GFX908-NEXT: s_nop 3 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-NEXT: s_nop 1 ; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_4x4x2bf16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v0, 1 ; GFX90A-NEXT: v_mov_b32_e32 v2, 2 ; GFX90A-NEXT: v_mov_b32_e32 v1, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: v_mfma_f32_4x4x2bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 4 ; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] ; GFX90A-NEXT: s_endpgm ; ; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x2bf16: ; GFX90A-VGPR: ; %bb.0: ; %bb ; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: v_mfma_f32_4x4x2bf16 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-VGPR-NEXT: s_nop 4 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] ; GFX90A-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> %b = bitcast i32 2 to <2 x i16> %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x2bf16(<2 x i16> %a, <2 x i16> %b, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_32x32x4bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-LABEL: test_mfma_f32_32x32x4bf16: ; GFX908-SDAG: ; %bb.0: ; %bb ; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX908-SDAG-NEXT: s_nop 15 ; GFX908-SDAG-NEXT: s_nop 1 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a3 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a2 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a1 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a0 ; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48 ; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32 ; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16 ; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] ; GFX908-SDAG-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4bf16: ; GFX908-GISEL: ; %bb.0: ; %bb ; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 ; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 ; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-GISEL-NEXT: s_nop 1 ; GFX908-GISEL-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX908-GISEL-NEXT: s_nop 15 ; GFX908-GISEL-NEXT: s_nop 1 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 ; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x4bf16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v0, 1 ; GFX90A-NEXT: v_mov_b32_e32 v1, 2 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: s_nop 15 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX90A-NEXT: s_endpgm ; ; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16: ; GFX90A-VGPR: ; %bb.0: ; %bb ; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 2 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x4bf16 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 0 ; GFX90A-VGPR-NEXT: s_nop 15 ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX90A-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> %b = bitcast i32 2 to <2 x i16> %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16> %a, <2 x i16> %b, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 { ; GFX908-LABEL: test_mfma_f32_16x16x8bf16: ; GFX908: ; %bb.0: ; %bb ; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX908-NEXT: v_mov_b32_e32 v0, 1 ; GFX908-NEXT: v_mov_b32_e32 v1, 2 ; GFX908-NEXT: v_mov_b32_e32 v4, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: v_mov_b32_e32 v5, s0 ; GFX908-NEXT: v_mov_b32_e32 v2, s1 ; GFX908-NEXT: v_mov_b32_e32 v3, s2 ; GFX908-NEXT: v_accvgpr_write_b32 a0, v5 ; GFX908-NEXT: v_mov_b32_e32 v5, s3 ; GFX908-NEXT: v_accvgpr_write_b32 a1, v2 ; GFX908-NEXT: v_accvgpr_write_b32 a2, v3 ; GFX908-NEXT: v_accvgpr_write_b32 a3, v5 ; GFX908-NEXT: s_nop 0 ; GFX908-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 ; GFX908-NEXT: s_nop 9 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 ; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 ; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 ; GFX908-NEXT: s_nop 1 ; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_16x16x8bf16: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v0, 1 ; GFX90A-NEXT: v_mov_b32_e32 v2, 2 ; GFX90A-NEXT: v_mov_b32_e32 v1, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-NEXT: s_nop 10 ; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] ; GFX90A-NEXT: s_endpgm ; ; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x8bf16: ; GFX90A-VGPR: ; %bb.0: ; %bb ; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2 ; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] ; GFX90A-VGPR-NEXT: s_nop 1 ; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x8bf16 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3 ; GFX90A-VGPR-NEXT: s_nop 10 ; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] ; GFX90A-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i32 1 to <2 x i16> %b = bitcast i32 2 to <2 x i16> %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16> %a, <2 x i16> %b, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }