; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck --check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck --check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9-NODL %s ; RUN: opt -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-codegenprepare -S < %s | FileCheck --check-prefixes=OPT %s ; Test dot4 patterns with saturating add (clamp) and without ;------------------------------------------------------------------------------ ; DOT4 TESTS WITH <4 x i8> ARGS AND VECTOR REDUCE ;------------------------------------------------------------------------------ define i32 @test_udot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: test_udot4_sat: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: test_udot4_sat: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: test_udot4_sat: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: test_udot4_sat: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1 ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v8 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %conv.i = zext <4 x i8> %a to <4 x i32> %conv8.i = zext <4 x i8> %b to <4 x i32> %mul.i = mul <4 x i32> %conv8.i, %conv.i %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) %cond.i.i = tail call i32 @llvm.uadd.sat.i32(i32 %rdx.add.i, i32 %c) ret i32 %cond.i.i } define i32 @test_udot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: test_udot4_unsat: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: test_udot4_unsat: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: test_udot4_unsat: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: test_udot4_unsat: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %conv.i = zext <4 x i8> %a to <4 x i32> %conv8.i = zext <4 x i8> %b to <4 x i32> %mul.i = mul <4 x i32> %conv8.i, %conv.i %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) %add.i = add i32 %rdx.add.i, %c ret i32 %add.i } define i32 @test_sdot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: test_sdot4_sat: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: test_sdot4_sat: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 ; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: test_sdot4_sat: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX950-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: test_sdot4_sat: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1 ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v8 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %conv.i = sext <4 x i8> %a to <4 x i32> %conv8.i = sext <4 x i8> %b to <4 x i32> %mul.i = mul <4 x i32> %conv8.i, %conv.i %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) %cond1.i.i = tail call i32 @llvm.sadd.sat.i32(i32 %rdx.add.i, i32 %c) ret i32 %cond1.i.i } define i32 @test_sdot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: test_sdot4_unsat: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: test_sdot4_unsat: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v2, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: test_sdot4_unsat: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 ; GFX950-NEXT: v_mov_b32_e32 v2, 0 ; GFX950-NEXT: v_dot4c_i32_i8_e32 v2, v1, v0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v2, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: test_sdot4_unsat: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %conv.i = sext <4 x i8> %a to <4 x i32> %conv8.i = sext <4 x i8> %b to <4 x i32> %mul.i = mul <4 x i32> %conv8.i, %conv.i %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) %add.i = add i32 %rdx.add.i, %c ret i32 %add.i } ;------------------------------------------------------------------------------ ; DOT4 TESTS WITH BITCAST FROM I32 ;------------------------------------------------------------------------------ ; Unsigned dot4 without saturation using bitcast from i32 ; OPT-LABEL: @udot4_unsat_bitcast( ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false) ; OPT: [[RES:%.*]] = add i32 [[DOT]], %c ; OPT: ret i32 [[RES]] ; define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: udot4_unsat_bitcast: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: udot4_unsat_bitcast: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v2 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: udot4_unsat_bitcast: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: udot4_unsat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = zext <4 x i8> %a_vec to <4 x i32> %b_ext = zext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = add i32 %sum, %c ret i32 %result } ; Signed dot4 without saturation using bitcast from i32 ; OPT-LABEL: @sdot4_unsat_bitcast( ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false) ; OPT: [[RES:%.*]] = add i32 [[DOT]], %c ; OPT: ret i32 [[RES]] ; define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: sdot4_unsat_bitcast: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: sdot4_unsat_bitcast: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0 ; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v2 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: sdot4_unsat_bitcast: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_mov_b32_e32 v3, 0 ; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v3, v2 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: sdot4_unsat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = sext <4 x i8> %a_vec to <4 x i32> %b_ext = sext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = add i32 %sum, %c ret i32 %result } ; Unsigned dot4 with saturation using bitcast from i32 ; OPT-LABEL: @udot4_sat_bitcast( ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true) ; OPT: ret i32 [[DOT]] ; define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: udot4_sat_bitcast: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: udot4_sat_bitcast: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: udot4_sat_bitcast: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: udot4_sat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = zext <4 x i8> %a_vec to <4 x i32> %b_ext = zext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c) ret i32 %result } ; Signed dot4 with saturation using bitcast from i32 ; OPT-LABEL: @sdot4_sat_bitcast( ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true) ; OPT: ret i32 [[DOT]] ; define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: sdot4_sat_bitcast: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: sdot4_sat_bitcast: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: sdot4_sat_bitcast: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: sdot4_sat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = sext <4 x i8> %a_vec to <4 x i32> %b_ext = sext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c) ret i32 %result } ;------------------------------------------------------------------------------ ; TESTS WITH <4 x i8> FUNCTION ARGUMENTS ;------------------------------------------------------------------------------ ; Test with raw <4 x i8> arguments (ABI passes as separate registers) define i32 @udot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: udot4_v4i8_arg: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: udot4_v4i8_arg: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: udot4_v4i8_arg: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: udot4_v4i8_arg: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_ext = zext <4 x i8> %a to <4 x i32> %b_ext = zext <4 x i8> %b to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = add i32 %sum, %c ret i32 %result } define i32 @sdot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) { ; GFX9-DL-LABEL: sdot4_v4i8_arg: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: sdot4_v4i8_arg: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4 ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: sdot4_v4i8_arg: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 ; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4 ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX950-NEXT: v_mov_b32_e32 v1, 0 ; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v1, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: sdot4_v4i8_arg: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_ext = sext <4 x i8> %a to <4 x i32> %b_ext = sext <4 x i8> %b to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = add i32 %sum, %c ret i32 %result } ;------------------------------------------------------------------------------ ; TESTS WITH I8 PROMOTION ;------------------------------------------------------------------------------ ; Test unsigned dot4 with i8 values zext'd individually define i32 @udot4_i8_zext(i8 %a0, i8 %a1, i8 %a2, i8 %a3, ; GFX9-DL-LABEL: udot4_i8_zext: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400 ; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5 ; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: udot4_i8_zext: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400 ; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c ; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: udot4_i8_zext: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400 ; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1 ; GFX950-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX950-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: udot4_i8_zext: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) { entry: %a0_32 = zext i8 %a0 to i32 %a1_32 = zext i8 %a1 to i32 %a2_32 = zext i8 %a2 to i32 %a3_32 = zext i8 %a3 to i32 %b0_32 = zext i8 %b0 to i32 %b1_32 = zext i8 %b1 to i32 %b2_32 = zext i8 %b2 to i32 %b3_32 = zext i8 %b3 to i32 %m0 = mul i32 %a0_32, %b0_32 %m1 = mul i32 %a1_32, %b1_32 %m2 = mul i32 %a2_32, %b2_32 %m3 = mul i32 %a3_32, %b3_32 %sum01 = add i32 %m0, %m1 %sum012 = add i32 %sum01, %m2 %sum0123 = add i32 %sum012, %m3 %result = add i32 %sum0123, %c ret i32 %result } ; Test signed dot4 with i8 values sext'd individually define i32 @sdot4_i8_sext(i8 %a0, i8 %a1, i8 %a2, i8 %a3, ; GFX9-DL-LABEL: sdot4_i8_sext: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400 ; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 ; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5 ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 ; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5 ; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: sdot4_i8_sext: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400 ; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c ; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: sdot4_i8_sext: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400 ; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 ; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1 ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 ; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1 ; GFX950-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX950-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX950-NEXT: v_mov_b32_e32 v1, 0 ; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e32 v0, v1, v8 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: sdot4_i8_sext: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) { entry: %a0_32 = sext i8 %a0 to i32 %a1_32 = sext i8 %a1 to i32 %a2_32 = sext i8 %a2 to i32 %a3_32 = sext i8 %a3 to i32 %b0_32 = sext i8 %b0 to i32 %b1_32 = sext i8 %b1 to i32 %b2_32 = sext i8 %b2 to i32 %b3_32 = sext i8 %b3 to i32 %m0 = mul i32 %a0_32, %b0_32 %m1 = mul i32 %a1_32, %b1_32 %m2 = mul i32 %a2_32, %b2_32 %m3 = mul i32 %a3_32, %b3_32 %sum01 = add i32 %m0, %m1 %sum012 = add i32 %sum01, %m2 %sum0123 = add i32 %sum012, %m3 %result = add i32 %sum0123, %c ret i32 %result } ;------------------------------------------------------------------------------ ; NEGATIVE TESTS ;------------------------------------------------------------------------------ ; Negative test: mismatched signedness (zext with sadd.sat) define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: dot4_mismatched_sign_zext_sadd: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX9-DL-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: dot4_mismatched_sign_zext_sadd: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX10-DL-NEXT: v_add_nc_i32 v0, v0, v2 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: dot4_mismatched_sign_zext_sadd: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = zext <4 x i8> %a_vec to <4 x i32> %b_ext = zext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c) ret i32 %result } ; Negative test: mismatched signedness (sext with uadd.sat) define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: dot4_mismatched_sign_sext_uadd: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0 ; GFX9-DL-NEXT: v_add_u32_e64 v0, v0, v2 clamp ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: dot4_mismatched_sign_sext_uadd: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0 ; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1 ; GFX10-DL-NEXT: v_add_nc_u32_e64 v0, v3, v2 clamp ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: dot4_mismatched_sign_sext_uadd: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_mov_b32_e32 v3, 0 ; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1 ; GFX950-NEXT: s_nop 2 ; GFX950-NEXT: v_add_u32_e64 v0, v3, v2 clamp ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = sext <4 x i8> %a_vec to <4 x i32> %b_ext = sext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c) ret i32 %result } ; Negative test: mixed zext/sext operands define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: dot4_mixed_ext: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5 ; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: dot4_mixed_ext: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff ; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: dot4_mixed_ext: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 ; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX950-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX950-NEXT: v_add_u32_e32 v1, v4, v5 ; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: dot4_mixed_ext: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: %a_vec = bitcast i32 %a_packed to <4 x i8> %b_vec = bitcast i32 %b_packed to <4 x i8> %a_ext = zext <4 x i8> %a_vec to <4 x i32> %b_ext = sext <4 x i8> %b_vec to <4 x i32> %mul = mul <4 x i32> %a_ext, %b_ext %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) %result = add i32 %sum, %c ret i32 %result } declare i32 @llvm.sadd.sat.i32(i32, i32) declare i32 @llvm.uadd.sat.i32(i32, i32) declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)