derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
945 lines
43 KiB
LLVM
945 lines
43 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck --check-prefixes=GFX9-DL %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck --check-prefixes=GFX10-DL %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9-NODL %s
|
|
; RUN: opt -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-codegenprepare -S < %s | FileCheck --check-prefixes=OPT %s
|
|
|
|
; Test dot4 patterns with saturating add (clamp) and without
|
|
|
|
;------------------------------------------------------------------------------
|
|
; DOT4 TESTS WITH <4 x i8> ARGS AND VECTOR REDUCE
|
|
;------------------------------------------------------------------------------
|
|
|
|
define i32 @test_udot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: test_udot4_sat:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: test_udot4_sat:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: test_udot4_sat:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: test_udot4_sat:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1
|
|
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v8 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%conv.i = zext <4 x i8> %a to <4 x i32>
|
|
%conv8.i = zext <4 x i8> %b to <4 x i32>
|
|
%mul.i = mul <4 x i32> %conv8.i, %conv.i
|
|
%rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
|
|
%cond.i.i = tail call i32 @llvm.uadd.sat.i32(i32 %rdx.add.i, i32 %c)
|
|
ret i32 %cond.i.i
|
|
}
|
|
|
|
define i32 @test_udot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: test_udot4_unsat:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: test_udot4_unsat:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: test_udot4_unsat:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: test_udot4_unsat:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%conv.i = zext <4 x i8> %a to <4 x i32>
|
|
%conv8.i = zext <4 x i8> %b to <4 x i32>
|
|
%mul.i = mul <4 x i32> %conv8.i, %conv.i
|
|
%rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
|
|
%add.i = add i32 %rdx.add.i, %c
|
|
ret i32 %add.i
|
|
}
|
|
|
|
define i32 @test_sdot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: test_sdot4_sat:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: test_sdot4_sat:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
|
|
; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: test_sdot4_sat:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX950-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: test_sdot4_sat:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1
|
|
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v8 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%conv.i = sext <4 x i8> %a to <4 x i32>
|
|
%conv8.i = sext <4 x i8> %b to <4 x i32>
|
|
%mul.i = mul <4 x i32> %conv8.i, %conv.i
|
|
%rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
|
|
%cond1.i.i = tail call i32 @llvm.sadd.sat.i32(i32 %rdx.add.i, i32 %c)
|
|
ret i32 %cond1.i.i
|
|
}
|
|
|
|
define i32 @test_sdot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: test_sdot4_unsat:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: test_sdot4_unsat:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
|
|
; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v2, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: test_sdot4_unsat:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1
|
|
; GFX950-NEXT: v_mov_b32_e32 v2, 0
|
|
; GFX950-NEXT: v_dot4c_i32_i8_e32 v2, v1, v0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v2, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: test_sdot4_unsat:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%conv.i = sext <4 x i8> %a to <4 x i32>
|
|
%conv8.i = sext <4 x i8> %b to <4 x i32>
|
|
%mul.i = mul <4 x i32> %conv8.i, %conv.i
|
|
%rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
|
|
%add.i = add i32 %rdx.add.i, %c
|
|
ret i32 %add.i
|
|
}
|
|
|
|
;------------------------------------------------------------------------------
|
|
; DOT4 TESTS WITH BITCAST FROM I32
|
|
;------------------------------------------------------------------------------
|
|
|
|
; Unsigned dot4 without saturation using bitcast from i32
|
|
; OPT-LABEL: @udot4_unsat_bitcast(
|
|
; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
|
|
; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
|
|
; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
|
|
; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
|
|
; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false)
|
|
; OPT: [[RES:%.*]] = add i32 [[DOT]], %c
|
|
; OPT: ret i32 [[RES]]
|
|
;
|
|
define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: udot4_unsat_bitcast:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: udot4_unsat_bitcast:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v2
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: udot4_unsat_bitcast:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: udot4_unsat_bitcast:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = zext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = zext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = add i32 %sum, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
; Signed dot4 without saturation using bitcast from i32
|
|
; OPT-LABEL: @sdot4_unsat_bitcast(
|
|
; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
|
|
; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
|
|
; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
|
|
; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
|
|
; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false)
|
|
; OPT: [[RES:%.*]] = add i32 [[DOT]], %c
|
|
; OPT: ret i32 [[RES]]
|
|
;
|
|
define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: sdot4_unsat_bitcast:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: sdot4_unsat_bitcast:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
|
|
; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v2
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: sdot4_unsat_bitcast:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_mov_b32_e32 v3, 0
|
|
; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v3, v2
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: sdot4_unsat_bitcast:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = sext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = sext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = add i32 %sum, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
; Unsigned dot4 with saturation using bitcast from i32
|
|
; OPT-LABEL: @udot4_sat_bitcast(
|
|
; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
|
|
; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
|
|
; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
|
|
; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
|
|
; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true)
|
|
; OPT: ret i32 [[DOT]]
|
|
;
|
|
define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: udot4_sat_bitcast:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: udot4_sat_bitcast:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: udot4_sat_bitcast:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: udot4_sat_bitcast:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
|
|
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = zext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = zext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Signed dot4 with saturation using bitcast from i32
|
|
; OPT-LABEL: @sdot4_sat_bitcast(
|
|
; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
|
|
; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
|
|
; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
|
|
; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
|
|
; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true)
|
|
; OPT: ret i32 [[DOT]]
|
|
;
|
|
define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: sdot4_sat_bitcast:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: sdot4_sat_bitcast:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: sdot4_sat_bitcast:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: sdot4_sat_bitcast:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
|
|
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = sext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = sext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c)
|
|
ret i32 %result
|
|
}
|
|
|
|
;------------------------------------------------------------------------------
|
|
; TESTS WITH <4 x i8> FUNCTION ARGUMENTS
|
|
;------------------------------------------------------------------------------
|
|
|
|
; Test with raw <4 x i8> arguments (ABI passes as separate registers)
|
|
define i32 @udot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: udot4_v4i8_arg:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: udot4_v4i8_arg:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: udot4_v4i8_arg:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: udot4_v4i8_arg:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_ext = zext <4 x i8> %a to <4 x i32>
|
|
%b_ext = zext <4 x i8> %b to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = add i32 %sum, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
define i32 @sdot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) {
|
|
; GFX9-DL-LABEL: sdot4_v4i8_arg:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004
|
|
; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
|
|
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: sdot4_v4i8_arg:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4
|
|
; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
|
|
; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: sdot4_v4i8_arg:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
|
|
; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
|
|
; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4
|
|
; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0
|
|
; GFX950-NEXT: v_mov_b32_e32 v1, 0
|
|
; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v1, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: sdot4_v4i8_arg:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_ext = sext <4 x i8> %a to <4 x i32>
|
|
%b_ext = sext <4 x i8> %b to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = add i32 %sum, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
;------------------------------------------------------------------------------
|
|
; TESTS WITH I8 PROMOTION
|
|
;------------------------------------------------------------------------------
|
|
|
|
; Test unsigned dot4 with i8 values zext'd individually
|
|
define i32 @udot4_i8_zext(i8 %a0, i8 %a1, i8 %a2, i8 %a3,
|
|
; GFX9-DL-LABEL: udot4_i8_zext:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400
|
|
; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c
|
|
; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5
|
|
; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4
|
|
; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: udot4_i8_zext:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400
|
|
; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c
|
|
; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4
|
|
; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: udot4_i8_zext:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400
|
|
; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c
|
|
; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1
|
|
; GFX950-NEXT: v_or_b32_e32 v4, v5, v4
|
|
; GFX950-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: udot4_i8_zext:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) {
|
|
entry:
|
|
%a0_32 = zext i8 %a0 to i32
|
|
%a1_32 = zext i8 %a1 to i32
|
|
%a2_32 = zext i8 %a2 to i32
|
|
%a3_32 = zext i8 %a3 to i32
|
|
%b0_32 = zext i8 %b0 to i32
|
|
%b1_32 = zext i8 %b1 to i32
|
|
%b2_32 = zext i8 %b2 to i32
|
|
%b3_32 = zext i8 %b3 to i32
|
|
%m0 = mul i32 %a0_32, %b0_32
|
|
%m1 = mul i32 %a1_32, %b1_32
|
|
%m2 = mul i32 %a2_32, %b2_32
|
|
%m3 = mul i32 %a3_32, %b3_32
|
|
%sum01 = add i32 %m0, %m1
|
|
%sum012 = add i32 %sum01, %m2
|
|
%sum0123 = add i32 %sum012, %m3
|
|
%result = add i32 %sum0123, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
; Test signed dot4 with i8 values sext'd individually
|
|
define i32 @sdot4_i8_sext(i8 %a0, i8 %a1, i8 %a2, i8 %a3,
|
|
; GFX9-DL-LABEL: sdot4_i8_sext:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400
|
|
; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c
|
|
; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5
|
|
; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4
|
|
; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5
|
|
; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4
|
|
; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: sdot4_i8_sext:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400
|
|
; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c
|
|
; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400
|
|
; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c
|
|
; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4
|
|
; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
|
|
; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: sdot4_i8_sext:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400
|
|
; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c
|
|
; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0
|
|
; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1
|
|
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
|
|
; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1
|
|
; GFX950-NEXT: v_or_b32_e32 v4, v5, v4
|
|
; GFX950-NEXT: v_or_b32_e32 v0, v1, v0
|
|
; GFX950-NEXT: v_mov_b32_e32 v1, 0
|
|
; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v1, v8
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: sdot4_i8_sext:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) {
|
|
entry:
|
|
%a0_32 = sext i8 %a0 to i32
|
|
%a1_32 = sext i8 %a1 to i32
|
|
%a2_32 = sext i8 %a2 to i32
|
|
%a3_32 = sext i8 %a3 to i32
|
|
%b0_32 = sext i8 %b0 to i32
|
|
%b1_32 = sext i8 %b1 to i32
|
|
%b2_32 = sext i8 %b2 to i32
|
|
%b3_32 = sext i8 %b3 to i32
|
|
%m0 = mul i32 %a0_32, %b0_32
|
|
%m1 = mul i32 %a1_32, %b1_32
|
|
%m2 = mul i32 %a2_32, %b2_32
|
|
%m3 = mul i32 %a3_32, %b3_32
|
|
%sum01 = add i32 %m0, %m1
|
|
%sum012 = add i32 %sum01, %m2
|
|
%sum0123 = add i32 %sum012, %m3
|
|
%result = add i32 %sum0123, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
;------------------------------------------------------------------------------
|
|
; NEGATIVE TESTS
|
|
;------------------------------------------------------------------------------
|
|
|
|
; Negative test: mismatched signedness (zext with sadd.sat)
|
|
define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: dot4_mismatched_sign_zext_sadd:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX9-DL-NEXT: v_add_i32 v0, v0, v2 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: dot4_mismatched_sign_zext_sadd:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX10-DL-NEXT: v_add_nc_i32 v0, v0, v2 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: dot4_mismatched_sign_zext_sadd:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_i32 v0, v0, v2 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
|
|
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = zext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = zext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Negative test: mismatched signedness (sext with uadd.sat)
|
|
define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: dot4_mismatched_sign_sext_uadd:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0
|
|
; GFX9-DL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: dot4_mismatched_sign_sext_uadd:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
|
|
; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e64 v0, v3, v2 clamp
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: dot4_mismatched_sign_sext_uadd:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_mov_b32_e32 v3, 0
|
|
; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1
|
|
; GFX950-NEXT: s_nop 2
|
|
; GFX950-NEXT: v_add_u32_e64 v0, v3, v2 clamp
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
|
|
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = sext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = sext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Negative test: mixed zext/sext operands
|
|
define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) {
|
|
; GFX9-DL-LABEL: dot4_mixed_ext:
|
|
; GFX9-DL: ; %bb.0: ; %entry
|
|
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
|
|
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5
|
|
; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2
|
|
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX10-DL-LABEL: dot4_mixed_ext:
|
|
; GFX10-DL: ; %bb.0: ; %entry
|
|
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff
|
|
; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
|
|
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
|
|
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0
|
|
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6
|
|
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
|
|
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX950-LABEL: dot4_mixed_ext:
|
|
; GFX950: ; %bb.0: ; %entry
|
|
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
|
|
; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX950-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX950-NEXT: v_add_u32_e32 v1, v4, v5
|
|
; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2
|
|
; GFX950-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-NODL-LABEL: dot4_mixed_ext:
|
|
; GFX9-NODL: ; %bb.0: ; %entry
|
|
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
|
|
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
|
|
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5
|
|
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
|
|
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%a_vec = bitcast i32 %a_packed to <4 x i8>
|
|
%b_vec = bitcast i32 %b_packed to <4 x i8>
|
|
%a_ext = zext <4 x i8> %a_vec to <4 x i32>
|
|
%b_ext = sext <4 x i8> %b_vec to <4 x i32>
|
|
%mul = mul <4 x i32> %a_ext, %b_ext
|
|
%sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
|
|
%result = add i32 %sum, %c
|
|
ret i32 %result
|
|
}
|
|
|
|
declare i32 @llvm.sadd.sat.i32(i32, i32)
|
|
declare i32 @llvm.uadd.sat.i32(i32, i32)
|
|
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
|