; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p3 < %s | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p3 -force-streaming < %s | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefix=CHECK-NOSVE2P3

define <vscale x 8 x i16> @udot_btoh_scalable(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: udot_btoh_scalable:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    udot z0.h, z1.b, z2.b
; CHECK-NEXT:    ret
;
; CHECK-NOSVE2P3-LABEL: udot_btoh_scalable:
; CHECK-NOSVE2P3:       // %bb.0: // %entry
; CHECK-NOSVE2P3-NEXT:    uunpklo z3.h, z1.b
; CHECK-NOSVE2P3-NEXT:    uunpklo z4.h, z2.b
; CHECK-NOSVE2P3-NEXT:    ptrue p0.h
; CHECK-NOSVE2P3-NEXT:    uunpkhi z1.h, z1.b
; CHECK-NOSVE2P3-NEXT:    uunpkhi z2.h, z2.b
; CHECK-NOSVE2P3-NEXT:    mla z0.h, p0/m, z3.h, z4.h
; CHECK-NOSVE2P3-NEXT:    mla z0.h, p0/m, z1.h, z2.h
; CHECK-NOSVE2P3-NEXT:    ret
entry:
  %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
  %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
  %mult = mul nuw nsw <vscale x 16 x i16> %a.wide, %b.wide
  %partial.reduce = tail call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %mult)
  ret <vscale x 8 x i16> %partial.reduce
}

define <vscale x 8 x i16> @sdot_btoh_scalable(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: sdot_btoh_scalable:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sdot z0.h, z1.b, z2.b
; CHECK-NEXT:    ret
;
; CHECK-NOSVE2P3-LABEL: sdot_btoh_scalable:
; CHECK-NOSVE2P3:       // %bb.0: // %entry
; CHECK-NOSVE2P3-NEXT:    sunpklo z3.h, z1.b
; CHECK-NOSVE2P3-NEXT:    sunpklo z4.h, z2.b
; CHECK-NOSVE2P3-NEXT:    ptrue p0.h
; CHECK-NOSVE2P3-NEXT:    sunpkhi z1.h, z1.b
; CHECK-NOSVE2P3-NEXT:    sunpkhi z2.h, z2.b
; CHECK-NOSVE2P3-NEXT:    mla z0.h, p0/m, z3.h, z4.h
; CHECK-NOSVE2P3-NEXT:    mla z0.h, p0/m, z1.h, z2.h
; CHECK-NOSVE2P3-NEXT:    ret
entry:
  %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i16>
  %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i16>
  %mult = mul nuw nsw <vscale x 16 x i16> %a.wide, %b.wide
  %partial.reduce = tail call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %mult)
  ret <vscale x 8 x i16> %partial.reduce
}