derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
296 lines
11 KiB
LLVM
296 lines
11 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: llc -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,SVE2
|
|
; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s --check-prefixes=CHECK,SVE2P1
|
|
|
|
target triple = "aarch64-linux-gnu"
|
|
|
|
define void @fdot_v4f32(ptr %accptr, ptr %aptr, ptr %bptr) {
|
|
; SVE2-LABEL: fdot_v4f32:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: ldr q0, [x0]
|
|
; SVE2-NEXT: ldr q1, [x1]
|
|
; SVE2-NEXT: ldr q2, [x2]
|
|
; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: str q0, [x0]
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fdot_v4f32:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: ldr q0, [x0]
|
|
; SVE2P1-NEXT: ldr q1, [x1]
|
|
; SVE2P1-NEXT: ldr q2, [x2]
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: str q0, [x0]
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%acc = load <4 x float>, ptr %accptr
|
|
%a = load <8 x half>, ptr %aptr
|
|
%b = load <8 x half>, ptr %bptr
|
|
%a.wide = fpext <8 x half> %a to <8 x float>
|
|
%b.wide = fpext <8 x half> %b to <8 x float>
|
|
%mult = fmul <8 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)
|
|
store <4 x float> %partial.reduce, ptr %accptr
|
|
ret void
|
|
}
|
|
|
|
define void @fdot_wide_v8f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(2,0) {
|
|
; SVE2-LABEL: fdot_wide_v8f32:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: ptrue p0.s, vl8
|
|
; SVE2-NEXT: mov x8, #8 // =0x8
|
|
; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]
|
|
; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]
|
|
; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
|
|
; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
|
|
; SVE2-NEXT: fcvt z0.s, p0/m, z0.h
|
|
; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
|
|
; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
|
|
; SVE2-NEXT: fcvt z3.s, p0/m, z3.h
|
|
; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]
|
|
; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s
|
|
; SVE2-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fdot_wide_v8f32:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: ptrue p0.s, vl8
|
|
; SVE2P1-NEXT: ptrue p1.h, vl16
|
|
; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]
|
|
; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]
|
|
; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%acc = load <8 x float>, ptr %accptr
|
|
%a = load <16 x half>, ptr %aptr
|
|
%b = load <16 x half>, ptr %bptr
|
|
%a.wide = fpext <16 x half> %a to <16 x float>
|
|
%b.wide = fpext <16 x half> %b to <16 x float>
|
|
%mult = fmul <16 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <8 x float> @llvm.vector.partial.reduce.fadd(<8 x float> %acc, <16 x float> %mult)
|
|
store <8 x float> %partial.reduce, ptr %accptr
|
|
ret void
|
|
}
|
|
|
|
define void @fdot_wide_v16f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(4,0) {
|
|
; SVE2-LABEL: fdot_wide_v16f32:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: ptrue p0.s, vl16
|
|
; SVE2-NEXT: mov x8, #16 // =0x10
|
|
; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]
|
|
; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]
|
|
; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
|
|
; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
|
|
; SVE2-NEXT: fcvt z0.s, p0/m, z0.h
|
|
; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
|
|
; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
|
|
; SVE2-NEXT: fcvt z3.s, p0/m, z3.h
|
|
; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]
|
|
; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s
|
|
; SVE2-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fdot_wide_v16f32:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: ptrue p0.s, vl16
|
|
; SVE2P1-NEXT: ptrue p1.h, vl32
|
|
; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]
|
|
; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]
|
|
; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%acc = load <16 x float>, ptr %accptr
|
|
%a = load <32 x half>, ptr %aptr
|
|
%b = load <32 x half>, ptr %bptr
|
|
%a.wide = fpext <32 x half> %a to <32 x float>
|
|
%b.wide = fpext <32 x half> %b to <32 x float>
|
|
%mult = fmul <32 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <16 x float> @llvm.vector.partial.reduce.fadd(<16 x float> %acc, <32 x float> %mult)
|
|
store <16 x float> %partial.reduce, ptr %accptr
|
|
ret void
|
|
}
|
|
|
|
define void @fdot_wide_v32f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(8,0) {
|
|
; SVE2-LABEL: fdot_wide_v32f32:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: ptrue p0.s, vl32
|
|
; SVE2-NEXT: mov x8, #32 // =0x20
|
|
; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]
|
|
; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]
|
|
; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
|
|
; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
|
|
; SVE2-NEXT: fcvt z0.s, p0/m, z0.h
|
|
; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
|
|
; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
|
|
; SVE2-NEXT: fcvt z3.s, p0/m, z3.h
|
|
; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]
|
|
; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s
|
|
; SVE2-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fdot_wide_v32f32:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: ptrue p0.s, vl32
|
|
; SVE2P1-NEXT: ptrue p1.h, vl64
|
|
; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]
|
|
; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]
|
|
; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%acc = load <32 x float>, ptr %accptr
|
|
%a = load <64 x half>, ptr %aptr
|
|
%b = load <64 x half>, ptr %bptr
|
|
%a.wide = fpext <64 x half> %a to <64 x float>
|
|
%b.wide = fpext <64 x half> %b to <64 x float>
|
|
%mult = fmul <64 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <32 x float> @llvm.vector.partial.reduce.fadd(<32 x float> %acc, <64 x float> %mult)
|
|
store <32 x float> %partial.reduce, ptr %accptr
|
|
ret void
|
|
}
|
|
|
|
define void @fdot_wide_v64f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(16,0) {
|
|
; SVE2-LABEL: fdot_wide_v64f32:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: ptrue p0.s, vl64
|
|
; SVE2-NEXT: mov x8, #64 // =0x40
|
|
; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]
|
|
; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]
|
|
; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
|
|
; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
|
|
; SVE2-NEXT: fcvt z0.s, p0/m, z0.h
|
|
; SVE2-NEXT: fcvt z1.s, p0/m, z1.h
|
|
; SVE2-NEXT: fcvt z2.s, p0/m, z2.h
|
|
; SVE2-NEXT: fcvt z3.s, p0/m, z3.h
|
|
; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]
|
|
; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s
|
|
; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s
|
|
; SVE2-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fdot_wide_v64f32:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: ptrue p0.s, vl64
|
|
; SVE2P1-NEXT: ptrue p1.h, vl128
|
|
; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]
|
|
; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]
|
|
; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%acc = load <64 x float>, ptr %accptr
|
|
%a = load <128 x half>, ptr %aptr
|
|
%b = load <128 x half>, ptr %bptr
|
|
%a.wide = fpext <128 x half> %a to <128 x float>
|
|
%b.wide = fpext <128 x half> %b to <128 x float>
|
|
%mult = fmul <128 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <64 x float> @llvm.vector.partial.reduce.fadd(<64 x float> %acc, <128 x float> %mult)
|
|
store <64 x float> %partial.reduce, ptr %accptr
|
|
ret void
|
|
}
|
|
|
|
define <4 x float> @fixed_fdot_wide(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {
|
|
; SVE2-LABEL: fixed_fdot_wide:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: // kill: def $q0 killed $q0 def $z0
|
|
; SVE2-NEXT: // kill: def $q2 killed $q2 def $z2
|
|
; SVE2-NEXT: // kill: def $q1 killed $q1 def $z1
|
|
; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fixed_fdot_wide:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: // kill: def $q0 killed $q0 def $z0
|
|
; SVE2P1-NEXT: // kill: def $q2 killed $q2 def $z2
|
|
; SVE2P1-NEXT: // kill: def $q1 killed $q1 def $z1
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: // kill: def $q0 killed $q0 killed $z0
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%a.wide = fpext <8 x half> %a to <8 x float>
|
|
%b.wide = fpext <8 x half> %b to <8 x float>
|
|
%mult = fmul <8 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)
|
|
ret <4 x float> %partial.reduce
|
|
}
|
|
|
|
define <2 x float> @fixed_fdot(<2 x float> %acc, <4 x half> %a, <4 x half> %b) {
|
|
; SVE2-LABEL: fixed_fdot:
|
|
; SVE2: // %bb.0: // %entry
|
|
; SVE2-NEXT: // kill: def $d0 killed $d0 def $z0
|
|
; SVE2-NEXT: // kill: def $d2 killed $d2 def $z2
|
|
; SVE2-NEXT: // kill: def $d1 killed $d1 def $z1
|
|
; SVE2-NEXT: fmlalb z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: fmlalt z0.s, z1.h, z2.h
|
|
; SVE2-NEXT: // kill: def $d0 killed $d0 killed $z0
|
|
; SVE2-NEXT: ret
|
|
;
|
|
; SVE2P1-LABEL: fixed_fdot:
|
|
; SVE2P1: // %bb.0: // %entry
|
|
; SVE2P1-NEXT: // kill: def $d0 killed $d0 def $z0
|
|
; SVE2P1-NEXT: // kill: def $d2 killed $d2 def $z2
|
|
; SVE2P1-NEXT: // kill: def $d1 killed $d1 def $z1
|
|
; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h
|
|
; SVE2P1-NEXT: // kill: def $d0 killed $d0 killed $z0
|
|
; SVE2P1-NEXT: ret
|
|
entry:
|
|
%a.wide = fpext <4 x half> %a to <4 x float>
|
|
%b.wide = fpext <4 x half> %b to <4 x float>
|
|
%mult = fmul <4 x float> %a.wide, %b.wide
|
|
%partial.reduce = call <2 x float> @llvm.vector.partial.reduce.fadd(<2 x float> %acc, <4 x float> %mult)
|
|
ret <2 x float> %partial.reduce
|
|
}
|
|
|
|
define <8 x half> @partial_reduce_half(<8 x half> %acc, <16 x half> %a) {
|
|
; CHECK-LABEL: partial_reduce_half:
|
|
; CHECK: // %bb.0: // %entry
|
|
; CHECK-NEXT: fadd v0.8h, v0.8h, v1.8h
|
|
; CHECK-NEXT: fadd v0.8h, v0.8h, v2.8h
|
|
; CHECK-NEXT: ret
|
|
entry:
|
|
%partial.reduce = call <8 x half> @llvm.vector.partial.reduce.fadd(<8 x half> %acc, <16 x half> %a)
|
|
ret <8 x half> %partial.reduce
|
|
}
|
|
|
|
define <4 x float> @partial_reduce_float(<4 x float> %acc, <8 x float> %a) {
|
|
; CHECK-LABEL: partial_reduce_float:
|
|
; CHECK: // %bb.0: // %entry
|
|
; CHECK-NEXT: fadd v0.4s, v0.4s, v1.4s
|
|
; CHECK-NEXT: fadd v0.4s, v0.4s, v2.4s
|
|
; CHECK-NEXT: ret
|
|
entry:
|
|
%partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %a)
|
|
ret <4 x float> %partial.reduce
|
|
}
|
|
|
|
define <2 x double> @partial_reduce_double(<2 x double> %acc, <4 x double> %a) {
|
|
; CHECK-LABEL: partial_reduce_double:
|
|
; CHECK: // %bb.0: // %entry
|
|
; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d
|
|
; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d
|
|
; CHECK-NEXT: ret
|
|
entry:
|
|
%partial.reduce = call <2 x double> @llvm.vector.partial.reduce.fadd(<2 x double> %acc, <4 x double> %a)
|
|
ret <2 x double> %partial.reduce
|
|
}
|