; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm    < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI

declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)
declare i32 @llvm.vector.reduce.add.v25i32(<25 x i32>)
declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)
declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>)
declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)
declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)

define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v4i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr s0, [x0]
; CHECK-SD-NEXT:    ldr s1, [x1]
; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    umull v0.4s, v1.4h, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v4i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr s0, [x1]
; CHECK-GI-NEXT:    ldr s1, [x0]
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    uaddlv s0, v0.4h
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <4 x i8>, ptr %a
  %1 = zext <4 x i8> %0 to <4 x i32>
  %2 = load <4 x i8>, ptr %b
  %3 = zext <4 x i8> %2 to <4 x i32>
  %4 = mul nuw nsw <4 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v4i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr s0, [x0]
; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    uaddlv s0, v0.4h
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v4i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr s0, [x0]
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    uaddlv s0, v0.4h
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    and w0, w8, #0xffff
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <4 x i8>, ptr %a1
  %1 = zext <4 x i8> %0 to <4 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v4i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr s0, [x0]
; CHECK-SD-NEXT:    ldr s1, [x1]
; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v4i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr s0, [x1]
; CHECK-GI-NEXT:    ldr s1, [x0]
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    saddlv s0, v0.4h
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <4 x i8>, ptr %a
  %1 = sext <4 x i8> %0 to <4 x i32>
  %2 = load <4 x i8>, ptr %b
  %3 = sext <4 x i8> %2 to <4 x i32>
  %4 = mul nsw <4 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v4i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    shl v2.4s, v2.4s, #24
; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #24
; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24
; CHECK-SD-NEXT:    sshr v2.4s, v2.4s, #24
; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #24
; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #24
; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24
; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT:    mla v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v2.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v4i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8
; CHECK-GI-NEXT:    shl v2.4h, v2.4h, #8
; CHECK-GI-NEXT:    shl v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-GI-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-GI-NEXT:    sshr v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v3.4h
; CHECK-GI-NEXT:    saddlv s0, v0.4h
; CHECK-GI-NEXT:    saddlv s1, v1.4h
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <4 x i8> %a to <4 x i32>
  %bz = sext <4 x i8> %b to <4 x i32>
  %m1 = mul nuw nsw <4 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
  %cz = sext <4 x i8> %c to <4 x i32>
  %dz = sext <4 x i8> %d to <4 x i32>
  %m2 = mul nuw nsw <4 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v4i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    ushll v1.4s, v2.4h, #0
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24
; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24
; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24
; CHECK-SD-NEXT:    ssra v0.4s, v1.4s, #24
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v4i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.4h, v2.4h, #8
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-GI-NEXT:    saddlv s1, v1.4h
; CHECK-GI-NEXT:    saddlv s0, v0.4h
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    sxth w8, w8
; CHECK-GI-NEXT:    add w0, w8, w9, sxth
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <4 x i8> %a to <4 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)
  %cz = sext <4 x i8> %c to <4 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_usdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_usdot_v4i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr s0, [x0]
; CHECK-SD-NEXT:    ldr s1, [x1]
; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v4i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr w8, [x0]
; CHECK-GI-NEXT:    ldr w9, [x1]
; CHECK-GI-NEXT:    fmov s0, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov b1, v0.b[1]
; CHECK-GI-NEXT:    mov b3, v0.b[2]
; CHECK-GI-NEXT:    mov b5, v2.b[2]
; CHECK-GI-NEXT:    mov b4, v0.b[3]
; CHECK-GI-NEXT:    mov b0, v2.b[1]
; CHECK-GI-NEXT:    mov b6, v2.b[3]
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    fmov w10, s1
; CHECK-GI-NEXT:    fmov w11, s3
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov w13, s5
; CHECK-GI-NEXT:    fmov w8, s4
; CHECK-GI-NEXT:    fmov w12, s0
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v1.h[1], w10
; CHECK-GI-NEXT:    fmov w10, s6
; CHECK-GI-NEXT:    fmov s0, w11
; CHECK-GI-NEXT:    fmov s3, w13
; CHECK-GI-NEXT:    mov v2.h[1], w12
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v0.h[1], w8
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    mov v3.h[1], w10
; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]
; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <4 x i8>, ptr %a
  %1 = zext <4 x i8> %0 to <4 x i32>
  %2 = load <4 x i8>, ptr %b
  %3 = sext <4 x i8> %2 to <4 x i32>
  %4 = mul nsw <4 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_usdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_v4i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #24
; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24
; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #24
; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #24
; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT:    mla v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v2.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v4i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0
; CHECK-GI-NEXT:    movi v4.2d, #0x0000ff000000ff
; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #24
; CHECK-GI-NEXT:    shl v3.4s, v3.4s, #24
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v4.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #24
; CHECK-GI-NEXT:    sshr v3.4s, v3.4s, #24
; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    mul v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <4 x i8> %a to <4 x i32>
  %bz = sext <4 x i8> %b to <4 x i32>
  %m1 = mul nuw nsw <4 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
  %cz = zext <4 x i8> %c to <4 x i32>
  %dz = sext <4 x i8> %d to <4 x i32>
  %m2 = mul nuw nsw <4 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v5i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr d0, [x0]
; CHECK-SD-NEXT:    ldr d1, [x1]
; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v5i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr d0, [x0]
; CHECK-GI-NEXT:    ldr d1, [x1]
; CHECK-GI-NEXT:    mov b2, v0.b[1]
; CHECK-GI-NEXT:    mov b3, v1.b[1]
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    mov b4, v1.b[2]
; CHECK-GI-NEXT:    mov b5, v0.b[2]
; CHECK-GI-NEXT:    mov b6, v0.b[3]
; CHECK-GI-NEXT:    mov b7, v1.b[3]
; CHECK-GI-NEXT:    mov b0, v0.b[4]
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    mov b1, v1.b[4]
; CHECK-GI-NEXT:    fmov w10, s3
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    fmov w11, s2
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov w8, s4
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    fmov w9, s5
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    mov v2.h[1], w10
; CHECK-GI-NEXT:    mov v3.h[1], w11
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    fmov w8, s7
; CHECK-GI-NEXT:    fmov w9, s6
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mul v0.8h, v2.8h, v3.8h
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[4]
; CHECK-GI-NEXT:    umov w10, v0.h[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    umov w8, v0.h[2]
; CHECK-GI-NEXT:    umov w9, v0.h[3]
; CHECK-GI-NEXT:    mov v1.s[1], w10
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v1.s[3], w9
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <5 x i8>, ptr %a
  %1 = zext <5 x i8> %0 to <5 x i32>
  %2 = load <5 x i8>, ptr %b
  %3 = zext <5 x i8> %2 to <5 x i32>
  %4 = mul nuw nsw <5 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v5i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr d0, [x0]
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v5i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr d0, [x0]
; CHECK-GI-NEXT:    umov w8, v0.b[0]
; CHECK-GI-NEXT:    umov w9, v0.b[4]
; CHECK-GI-NEXT:    umov w10, v0.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    umov w8, v0.b[2]
; CHECK-GI-NEXT:    umov w9, v0.b[3]
; CHECK-GI-NEXT:    mov v1.s[1], w10
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v1.s[3], w9
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <5 x i8>, ptr %a1
  %1 = zext <5 x i8> %0 to <5 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v5i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr d0, [x0]
; CHECK-SD-NEXT:    ldr d1, [x1]
; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v5i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr d0, [x0]
; CHECK-GI-NEXT:    ldr d1, [x1]
; CHECK-GI-NEXT:    mov b2, v0.b[1]
; CHECK-GI-NEXT:    mov b3, v1.b[1]
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    mov b4, v1.b[2]
; CHECK-GI-NEXT:    mov b5, v0.b[2]
; CHECK-GI-NEXT:    mov b6, v0.b[3]
; CHECK-GI-NEXT:    mov b7, v1.b[3]
; CHECK-GI-NEXT:    mov b0, v0.b[4]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov b1, v1.b[4]
; CHECK-GI-NEXT:    fmov w10, s3
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    fmov w11, s2
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov w8, s4
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    fmov w9, s5
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v2.h[1], w10
; CHECK-GI-NEXT:    mov v3.h[1], w11
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    fmov w8, s7
; CHECK-GI-NEXT:    fmov w9, s6
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mul v0.8h, v2.8h, v3.8h
; CHECK-GI-NEXT:    smov w8, v0.h[0]
; CHECK-GI-NEXT:    smov w9, v0.h[4]
; CHECK-GI-NEXT:    smov w10, v0.h[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    smov w8, v0.h[2]
; CHECK-GI-NEXT:    smov w9, v0.h[3]
; CHECK-GI-NEXT:    mov v1.s[1], w10
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v1.s[3], w9
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <5 x i8>, ptr %a
  %1 = sext <5 x i8> %0 to <5 x i32>
  %2 = load <5 x i8>, ptr %b
  %3 = sext <5 x i8> %2 to <5 x i32>
  %4 = mul nsw <5 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v5i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0
; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
; CHECK-SD-NEXT:    mov v1.s[0], v5.s[0]
; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v5i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    mov b17, v0.b[1]
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    fmov w11, s1
; CHECK-GI-NEXT:    mov b25, v1.b[1]
; CHECK-GI-NEXT:    mov b16, v1.b[2]
; CHECK-GI-NEXT:    mov b7, v1.b[3]
; CHECK-GI-NEXT:    mov b5, v1.b[4]
; CHECK-GI-NEXT:    mov b22, v2.b[1]
; CHECK-GI-NEXT:    mov b23, v3.b[1]
; CHECK-GI-NEXT:    sxtb w9, w8
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov b24, v0.b[2]
; CHECK-GI-NEXT:    fmov w8, s17
; CHECK-GI-NEXT:    mov b6, v0.b[3]
; CHECK-GI-NEXT:    mov b4, v0.b[4]
; CHECK-GI-NEXT:    fmov s1, w9
; CHECK-GI-NEXT:    mov b18, v2.b[2]
; CHECK-GI-NEXT:    mov b19, v2.b[3]
; CHECK-GI-NEXT:    mov b0, v2.b[4]
; CHECK-GI-NEXT:    fmov w9, s25
; CHECK-GI-NEXT:    fmov w12, s22
; CHECK-GI-NEXT:    sxtb w10, w8
; CHECK-GI-NEXT:    mov b21, v3.b[2]
; CHECK-GI-NEXT:    fmov w13, s23
; CHECK-GI-NEXT:    mov b20, v3.b[3]
; CHECK-GI-NEXT:    mov b17, v3.b[4]
; CHECK-GI-NEXT:    fmov w8, s24
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v1.h[1], w10
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    fmov w10, s2
; CHECK-GI-NEXT:    fmov s2, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    fmov w11, s3
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v2.h[1], w9
; CHECK-GI-NEXT:    fmov w9, s16
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v1.h[2], w8
; CHECK-GI-NEXT:    fmov w8, s7
; CHECK-GI-NEXT:    fmov s3, w10
; CHECK-GI-NEXT:    fmov w10, s18
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    fmov s22, w11
; CHECK-GI-NEXT:    fmov w11, s21
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v3.h[1], w12
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    mov v22.h[1], w13
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    fmov w9, s19
; CHECK-GI-NEXT:    fmov w12, s6
; CHECK-GI-NEXT:    mov v3.h[2], w10
; CHECK-GI-NEXT:    fmov w10, s20
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v22.h[2], w11
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    fmov w11, s4
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    fmov w8, s5
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v1.h[3], w12
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v22.h[3], w10
; CHECK-GI-NEXT:    fmov w10, s17
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v1.h[4], w11
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mov v22.h[4], w10
; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    mul v1.8h, v3.8h, v22.8h
; CHECK-GI-NEXT:    smov w8, v0.h[0]
; CHECK-GI-NEXT:    smov w9, v0.h[4]
; CHECK-GI-NEXT:    smov w11, v0.h[1]
; CHECK-GI-NEXT:    smov w10, v1.h[0]
; CHECK-GI-NEXT:    smov w12, v1.h[4]
; CHECK-GI-NEXT:    smov w13, v1.h[1]
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    smov w8, v0.h[2]
; CHECK-GI-NEXT:    smov w9, v1.h[2]
; CHECK-GI-NEXT:    fmov s4, w10
; CHECK-GI-NEXT:    fmov s5, w12
; CHECK-GI-NEXT:    mov v2.s[1], w11
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    smov w10, v0.h[3]
; CHECK-GI-NEXT:    smov w11, v1.h[3]
; CHECK-GI-NEXT:    mov v4.s[1], w13
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[2], w8
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], w9
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v2.s[3], w10
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], w11
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    add v1.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <5 x i8> %a to <5 x i32>
  %bz = sext <5 x i8> %b to <5 x i32>
  %m1 = mul nuw nsw <5 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
  %cz = sext <5 x i8> %c to <5 x i32>
  %dz = sext <5 x i8> %d to <5 x i32>
  %m2 = mul nuw nsw <5 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v5i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    sshll v1.8h, v2.8b, #0
; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
; CHECK-SD-NEXT:    mov v2.s[0], v5.s[0]
; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
; CHECK-SD-NEXT:    saddw v1.4s, v2.4s, v1.4h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v5i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT:    smov w8, v0.b[0]
; CHECK-GI-NEXT:    smov w9, v0.b[4]
; CHECK-GI-NEXT:    smov w10, v2.b[0]
; CHECK-GI-NEXT:    smov w12, v2.b[4]
; CHECK-GI-NEXT:    smov w11, v0.b[1]
; CHECK-GI-NEXT:    smov w13, v2.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    fmov s4, w10
; CHECK-GI-NEXT:    fmov s5, w12
; CHECK-GI-NEXT:    smov w8, v0.b[2]
; CHECK-GI-NEXT:    smov w9, v2.b[2]
; CHECK-GI-NEXT:    smov w10, v0.b[3]
; CHECK-GI-NEXT:    mov v1.s[1], w11
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    mov v4.s[1], w13
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    smov w11, v2.b[3]
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], w9
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v1.s[3], w10
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], w11
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    add v1.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <5 x i8> %a to <5 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)
  %cz = sext <5 x i8> %c to <5 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-LABEL: test_udot_v8i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.2d, #0000000000000000
; CHECK-NEXT:    ldr d1, [x0]
; CHECK-NEXT:    ldr d2, [x1]
; CHECK-NEXT:    udot v0.2s, v2.8b, v1.8b
; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a
  %1 = zext <8 x i8> %0 to <8 x i32>
  %2 = load <8 x i8>, ptr %b
  %3 = zext <8 x i8> %2 to <8 x i32>
  %4 = mul nuw nsw <8 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
  ret i32 %5
}

define i32 @test_udot_v8i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v8i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.8b, #1
; CHECK-SD-NEXT:    ldr d2, [x0]
; CHECK-SD-NEXT:    udot v0.2s, v2.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v8i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.8b, #1
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr d2, [x0]
; CHECK-GI-NEXT:    udot v1.2s, v2.8b, v0.8b
; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a1
  %1 = zext <8 x i8> %0 to <8 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
  ret i32 %2
}

define i32 @test_sdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-LABEL: test_sdot_v8i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.2d, #0000000000000000
; CHECK-NEXT:    ldr d1, [x0]
; CHECK-NEXT:    ldr d2, [x1]
; CHECK-NEXT:    sdot v0.2s, v2.8b, v1.8b
; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a
  %1 = sext <8 x i8> %0 to <8 x i32>
  %2 = load <8 x i8>, ptr %b
  %3 = sext <8 x i8> %2 to <8 x i32>
  %4 = mul nsw <8 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
  ret i32 %5
}

define i32 @test_sdot_v8i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_sdot_v8i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.8b, #1
; CHECK-SD-NEXT:    ldr d2, [x0]
; CHECK-SD-NEXT:    sdot v0.2s, v2.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v8i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.8b, #1
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr d2, [x0]
; CHECK-GI-NEXT:    sdot v1.2s, v2.8b, v0.8b
; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a1
  %1 = sext <8 x i8> %0 to <8 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
  ret i32 %2
}

define i32 @test_usdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-SD-LABEL: test_usdot_v8i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr d1, [x0]
; CHECK-SD-NEXT:    ldr d2, [x1]
; CHECK-SD-NEXT:    usdot v0.2s, v1.8b, v2.8b
; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v8i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr d0, [x0]
; CHECK-GI-NEXT:    ldr d1, [x1]
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    smull v2.4s, v1.4h, v0.4h
; CHECK-GI-NEXT:    smlal2 v2.4s, v1.8h, v0.8h
; CHECK-GI-NEXT:    addv s0, v2.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a
  %1 = zext <8 x i8> %0 to <8 x i32>
  %2 = load <8 x i8>, ptr %b
  %3 = sext <8 x i8> %2 to <8 x i32>
  %4 = mul nsw <8 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
  ret i32 %5
}

define i32 @test_usdot_swapped_operands_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr d1, [x0]
; CHECK-SD-NEXT:    ldr d2, [x1]
; CHECK-SD-NEXT:    usdot v0.2s, v2.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr d0, [x0]
; CHECK-GI-NEXT:    ldr d1, [x1]
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    smull v2.4s, v1.4h, v0.4h
; CHECK-GI-NEXT:    smlal2 v2.4s, v1.8h, v0.8h
; CHECK-GI-NEXT:    addv s0, v2.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <8 x i8>, ptr %a
  %1 = sext <8 x i8> %0 to <8 x i32>
  %2 = load <8 x i8>, ptr %b
  %3 = zext <8 x i8> %2 to <8 x i32>
  %4 = mul nsw <8 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
  ret i32 %5
}

define i32 @test_udot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v16i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.2d, #0000000000000000
; CHECK-NEXT:    ldr q1, [x0]
; CHECK-NEXT:    ldr q2, [x1]
; CHECK-NEXT:    udot v0.4s, v2.16b, v1.16b
; CHECK-NEXT:    addv s0, v0.4s
; CHECK-NEXT:    fmov w8, s0
; CHECK-NEXT:    add w0, w8, w2
; CHECK-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a
  %1 = zext <16 x i8> %0 to <16 x i32>
  %2 = load <16 x i8>, ptr %b
  %3 = zext <16 x i8> %2 to <16 x i32>
  %4 = mul nuw nsw <16 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v16i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_udot_v16i8_nomla:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.16b, #1
; CHECK-NEXT:    movi v1.2d, #0000000000000000
; CHECK-NEXT:    ldr q2, [x0]
; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
; CHECK-NEXT:    addv s0, v1.4s
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a1
  %1 = zext <16 x i8> %0 to <16 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
  ret i32 %2
}

define i32 @test_sdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_sdot_v16i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.2d, #0000000000000000
; CHECK-NEXT:    ldr q1, [x0]
; CHECK-NEXT:    ldr q2, [x1]
; CHECK-NEXT:    sdot v0.4s, v2.16b, v1.16b
; CHECK-NEXT:    addv s0, v0.4s
; CHECK-NEXT:    fmov w8, s0
; CHECK-NEXT:    add w0, w8, w2
; CHECK-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a
  %1 = sext <16 x i8> %0 to <16 x i32>
  %2 = load <16 x i8>, ptr %b
  %3 = sext <16 x i8> %2 to <16 x i32>
  %4 = mul nsw <16 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v16i8_nomla(ptr nocapture readonly %a1) {
; CHECK-LABEL: test_sdot_v16i8_nomla:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.16b, #1
; CHECK-NEXT:    movi v1.2d, #0000000000000000
; CHECK-NEXT:    ldr q2, [x0]
; CHECK-NEXT:    sdot v1.4s, v2.16b, v0.16b
; CHECK-NEXT:    addv s0, v1.4s
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a1
  %1 = sext <16 x i8> %0 to <16 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
  ret i32 %2
}

define i32 @test_usdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_usdot_v16i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q1, [x0]
; CHECK-SD-NEXT:    ldr q2, [x1]
; CHECK-SD-NEXT:    usdot v0.4s, v1.16b, v2.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v16i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr q0, [x0]
; CHECK-GI-NEXT:    ldr q1, [x1]
; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    sshll v3.8h, v1.8b, #0
; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    smull v4.4s, v3.4h, v2.4h
; CHECK-GI-NEXT:    smull v5.4s, v1.4h, v0.4h
; CHECK-GI-NEXT:    smlal2 v4.4s, v3.8h, v2.8h
; CHECK-GI-NEXT:    smlal2 v5.4s, v1.8h, v0.8h
; CHECK-GI-NEXT:    add v0.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a
  %1 = zext <16 x i8> %0 to <16 x i32>
  %2 = load <16 x i8>, ptr %b
  %3 = sext <16 x i8> %2 to <16 x i32>
  %4 = mul nsw <16 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_usdot_swapped_operands_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q1, [x0]
; CHECK-SD-NEXT:    ldr q2, [x1]
; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr q0, [x0]
; CHECK-GI-NEXT:    ldr q1, [x1]
; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    ushll v3.8h, v1.8b, #0
; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    smull v4.4s, v3.4h, v2.4h
; CHECK-GI-NEXT:    smull v5.4s, v1.4h, v0.4h
; CHECK-GI-NEXT:    smlal2 v4.4s, v3.8h, v2.8h
; CHECK-GI-NEXT:    smlal2 v5.4s, v1.8h, v0.8h
; CHECK-GI-NEXT:    add v0.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <16 x i8>, ptr %a
  %1 = sext <16 x i8> %0 to <16 x i32>
  %2 = load <16 x i8>, ptr %b
  %3 = zext <16 x i8> %2 to <16 x i32>
  %4 = mul nsw <16 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_udot_v8i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    udot v4.2s, v2.8b, v3.8b
; CHECK-SD-NEXT:    udot v4.2s, v0.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v8i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    udot v5.2s, v0.8b, v1.8b
; CHECK-GI-NEXT:    udot v4.2s, v2.8b, v3.8b
; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s
; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <8 x i8> %a to <8 x i32>
  %bz = zext <8 x i8> %b to <8 x i32>
  %m1 = mul nuw nsw <8 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
  %cz = zext <8 x i8> %c to <8 x i32>
  %dz = zext <8 x i8> %d to <8 x i32>
  %m2 = mul nuw nsw <8 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_udot_v8i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v3.8b, #1
; CHECK-SD-NEXT:    udot v1.2s, v2.8b, v3.8b
; CHECK-SD-NEXT:    udot v1.2s, v0.8b, v3.8b
; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v8i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v1.8b, #1
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    udot v4.2s, v0.8b, v1.8b
; CHECK-GI-NEXT:    udot v3.2s, v2.8b, v1.8b
; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <8 x i8> %a to <8 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
  %cz = zext <8 x i8> %c to <8 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_udot_v16i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    udot v4.4s, v2.16b, v3.16b
; CHECK-SD-NEXT:    udot v4.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v4.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v16i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    udot v5.4s, v0.16b, v1.16b
; CHECK-GI-NEXT:    udot v4.4s, v2.16b, v3.16b
; CHECK-GI-NEXT:    addv s0, v5.4s
; CHECK-GI-NEXT:    addv s1, v4.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <16 x i8> %a to <16 x i32>
  %bz = zext <16 x i8> %b to <16 x i32>
  %m1 = mul nuw nsw <16 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
  %cz = zext <16 x i8> %c to <16 x i32>
  %dz = zext <16 x i8> %d to <16 x i32>
  %m2 = mul nuw nsw <16 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_udot_v16i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v1.16b, #1
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    udot v3.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    udot v3.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v3.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v16i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v1.16b, #1
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    udot v4.4s, v0.16b, v1.16b
; CHECK-GI-NEXT:    udot v3.4s, v2.16b, v1.16b
; CHECK-GI-NEXT:    addv s0, v4.4s
; CHECK-GI-NEXT:    addv s1, v3.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <16 x i8> %a to <16 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
  %cz = zext <16 x i8> %c to <16 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v8i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v4.2s, v2.8b, v3.8b
; CHECK-SD-NEXT:    sdot v4.2s, v0.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v8i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v5.2s, v0.8b, v1.8b
; CHECK-GI-NEXT:    sdot v4.2s, v2.8b, v3.8b
; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s
; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <8 x i8> %a to <8 x i32>
  %bz = sext <8 x i8> %b to <8 x i32>
  %m1 = mul nuw nsw <8 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
  %cz = sext <8 x i8> %c to <8 x i32>
  %dz = sext <8 x i8> %d to <8 x i32>
  %m2 = mul nuw nsw <8 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v8i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v3.8b, #1
; CHECK-SD-NEXT:    sdot v1.2s, v2.8b, v3.8b
; CHECK-SD-NEXT:    sdot v1.2s, v0.8b, v3.8b
; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v8i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v1.8b, #1
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v4.2s, v0.8b, v1.8b
; CHECK-GI-NEXT:    sdot v3.2s, v2.8b, v1.8b
; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <8 x i8> %a to <8 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
  %cz = sext <8 x i8> %c to <8 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v16i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v4.4s, v2.16b, v3.16b
; CHECK-SD-NEXT:    sdot v4.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v4.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v16i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v1.16b
; CHECK-GI-NEXT:    sdot v4.4s, v2.16b, v3.16b
; CHECK-GI-NEXT:    addv s0, v5.4s
; CHECK-GI-NEXT:    addv s1, v4.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <16 x i8> %a to <16 x i32>
  %bz = sext <16 x i8> %b to <16 x i32>
  %m1 = mul nuw nsw <16 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
  %cz = sext <16 x i8> %c to <16 x i32>
  %dz = sext <16 x i8> %d to <16 x i32>
  %m2 = mul nuw nsw <16 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v16i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v1.16b, #1
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v3.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v3.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v16i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v1.16b, #1
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v4.4s, v0.16b, v1.16b
; CHECK-GI-NEXT:    sdot v3.4s, v2.16b, v1.16b
; CHECK-GI-NEXT:    addv s0, v4.4s
; CHECK-GI-NEXT:    addv s1, v3.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <16 x i8> %a to <16 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
  %cz = sext <16 x i8> %c to <16 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}


define i32 @test_usdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_v8i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    usdot v4.2s, v2.8b, v3.8b
; CHECK-SD-NEXT:    usdot v4.2s, v0.8b, v1.8b
; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v8i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll v3.8h, v3.8b, #0
; CHECK-GI-NEXT:    smull v4.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    smull v5.4s, v2.4h, v3.4h
; CHECK-GI-NEXT:    smlal2 v4.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    smlal2 v5.4s, v2.8h, v3.8h
; CHECK-GI-NEXT:    addv s0, v4.4s
; CHECK-GI-NEXT:    addv s1, v5.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <8 x i8> %a to <8 x i32>
  %bz = sext <8 x i8> %b to <8 x i32>
  %m1 = mul nuw nsw <8 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
  %cz = zext <8 x i8> %c to <8 x i32>
  %dz = sext <8 x i8> %d to <8 x i32>
  %m2 = mul nuw nsw <8 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_usdot_swapped_operands_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    usdot v4.2s, v3.8b, v2.8b
; CHECK-SD-NEXT:    usdot v4.2s, v1.8b, v0.8b
; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    sshll v2.8h, v2.8b, #0
; CHECK-GI-NEXT:    ushll v3.8h, v3.8b, #0
; CHECK-GI-NEXT:    smull v4.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    smull v5.4s, v2.4h, v3.4h
; CHECK-GI-NEXT:    smlal2 v4.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    smlal2 v5.4s, v2.8h, v3.8h
; CHECK-GI-NEXT:    addv s0, v4.4s
; CHECK-GI-NEXT:    addv s1, v5.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <8 x i8> %a to <8 x i32>
  %bz = zext <8 x i8> %b to <8 x i32>
  %m1 = mul nuw nsw <8 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
  %cz = sext <8 x i8> %c to <8 x i32>
  %dz = zext <8 x i8> %d to <8 x i32>
  %m2 = mul nuw nsw <8 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_usdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_v16i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    usdot v4.4s, v2.16b, v3.16b
; CHECK-SD-NEXT:    usdot v4.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v4.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v16i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v4.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    sshll v5.8h, v1.8b, #0
; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    ushll v6.8h, v2.8b, #0
; CHECK-GI-NEXT:    ushll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    sshll v7.8h, v3.8b, #0
; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v5.4h
; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    smull v18.4s, v6.4h, v7.4h
; CHECK-GI-NEXT:    smull v19.4s, v2.4h, v3.4h
; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v5.8h
; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    smlal2 v18.4s, v6.8h, v7.8h
; CHECK-GI-NEXT:    smlal2 v19.4s, v2.8h, v3.8h
; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <16 x i8> %a to <16 x i32>
  %bz = sext <16 x i8> %b to <16 x i32>
  %m1 = mul nuw nsw <16 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
  %cz = zext <16 x i8> %c to <16 x i32>
  %dz = sext <16 x i8> %d to <16 x i32>
  %m2 = mul nuw nsw <16 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}


define i32 @test_usdot_swapped_operands_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    usdot v4.4s, v3.16b, v2.16b
; CHECK-SD-NEXT:    usdot v4.4s, v1.16b, v0.16b
; CHECK-SD-NEXT:    addv s0, v4.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v4.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    ushll v5.8h, v1.8b, #0
; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    sshll v6.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    ushll v7.8h, v3.8b, #0
; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v5.4h
; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    smull v18.4s, v6.4h, v7.4h
; CHECK-GI-NEXT:    smull v19.4s, v2.4h, v3.4h
; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v5.8h
; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    smlal2 v18.4s, v6.8h, v7.8h
; CHECK-GI-NEXT:    smlal2 v19.4s, v2.8h, v3.8h
; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <16 x i8> %a to <16 x i32>
  %bz = zext <16 x i8> %b to <16 x i32>
  %m1 = mul nuw nsw <16 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
  %cz = sext <16 x i8> %c to <16 x i32>
  %dz = zext <16 x i8> %d to <16 x i32>
  %m2 = mul nuw nsw <16 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v24i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q2, [x0]
; CHECK-SD-NEXT:    ldr q3, [x1]
; CHECK-SD-NEXT:    ldr d4, [x0, #16]
; CHECK-SD-NEXT:    ldr d5, [x1, #16]
; CHECK-SD-NEXT:    udot v1.2s, v5.8b, v4.8b
; CHECK-SD-NEXT:    udot v0.4s, v3.16b, v2.16b
; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s1
; CHECK-SD-NEXT:    fmov w9, s0
; CHECK-SD-NEXT:    add w8, w9, w8
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v24i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q2, [x0]
; CHECK-GI-NEXT:    ldr d3, [x0, #16]
; CHECK-GI-NEXT:    ldr q4, [x1]
; CHECK-GI-NEXT:    ldr d5, [x1, #16]
; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <24 x i8>, ptr %a
  %1 = zext <24 x i8> %0 to <24 x i32>
  %2 = load <24 x i8>, ptr %b
  %3 = zext <24 x i8> %2 to <24 x i32>
  %4 = mul nuw nsw <24 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v24i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.8b, #1
; CHECK-SD-NEXT:    ldr q4, [x0]
; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v3.16b, #1
; CHECK-SD-NEXT:    ldr d5, [x0, #16]
; CHECK-SD-NEXT:    udot v2.2s, v5.8b, v1.8b
; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
; CHECK-SD-NEXT:    addp v1.2s, v2.2s, v2.2s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s1
; CHECK-SD-NEXT:    fmov w9, s0
; CHECK-SD-NEXT:    add w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v24i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.8b, #1
; CHECK-GI-NEXT:    movi v1.8b, #1
; CHECK-GI-NEXT:    ldr q4, [x0]
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr d5, [x0, #16]
; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v0.16b
; CHECK-GI-NEXT:    udot v3.4s, v4.16b, v1.16b
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <24 x i8>, ptr %a1
  %1 = zext <24 x i8> %0 to <24 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v24i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q2, [x0]
; CHECK-SD-NEXT:    ldr q3, [x1]
; CHECK-SD-NEXT:    ldr d4, [x0, #16]
; CHECK-SD-NEXT:    ldr d5, [x1, #16]
; CHECK-SD-NEXT:    sdot v1.2s, v5.8b, v4.8b
; CHECK-SD-NEXT:    sdot v0.4s, v3.16b, v2.16b
; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s1
; CHECK-SD-NEXT:    fmov w9, s0
; CHECK-SD-NEXT:    add w8, w9, w8
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v24i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q2, [x0]
; CHECK-GI-NEXT:    ldr d3, [x0, #16]
; CHECK-GI-NEXT:    ldr q4, [x1]
; CHECK-GI-NEXT:    ldr d5, [x1, #16]
; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b
; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <24 x i8>, ptr %a
  %1 = sext <24 x i8> %0 to <24 x i32>
  %2 = load <24 x i8>, ptr %b
  %3 = sext <24 x i8> %2 to <24 x i32>
  %4 = mul nsw <24 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v24i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    ldr b1, [sp, #144]
; CHECK-SD-NEXT:    add x10, sp, #152
; CHECK-SD-NEXT:    add x9, sp, #160
; CHECK-SD-NEXT:    add x8, sp, #168
; CHECK-SD-NEXT:    ldr b2, [sp, #272]
; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
; CHECK-SD-NEXT:    add x11, sp, #280
; CHECK-SD-NEXT:    ldr b3, [sp, #80]
; CHECK-SD-NEXT:    mov v0.b[1], w1
; CHECK-SD-NEXT:    ldr b4, [sp, #528]
; CHECK-SD-NEXT:    add x10, sp, #88
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #536
; CHECK-SD-NEXT:    ldr b5, [sp, #336]
; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]
; CHECK-SD-NEXT:    add x10, sp, #344
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #176
; CHECK-SD-NEXT:    ldr b6, [sp, #656]
; CHECK-SD-NEXT:    mov v0.b[2], w2
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
; CHECK-SD-NEXT:    ldr b7, [sp, #464]
; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
; CHECK-SD-NEXT:    add x12, sp, #664
; CHECK-SD-NEXT:    add x9, sp, #472
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
; CHECK-SD-NEXT:    add x8, sp, #96
; CHECK-SD-NEXT:    add x10, sp, #184
; CHECK-SD-NEXT:    add x12, sp, #288
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
; CHECK-SD-NEXT:    mov v0.b[3], w3
; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x11]
; CHECK-SD-NEXT:    add x8, sp, #352
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]
; CHECK-SD-NEXT:    add x13, sp, #544
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #672
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x13]
; CHECK-SD-NEXT:    add x9, sp, #192
; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #480
; CHECK-SD-NEXT:    mov v0.b[4], w4
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #296
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #552
; CHECK-SD-NEXT:    add x12, sp, #200
; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #360
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #104
; CHECK-SD-NEXT:    add x9, sp, #560
; CHECK-SD-NEXT:    mov v0.b[5], w5
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #368
; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x12]
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]
; CHECK-SD-NEXT:    add x13, sp, #208
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
; CHECK-SD-NEXT:    add x12, sp, #304
; CHECK-SD-NEXT:    add x8, sp, #568
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
; CHECK-SD-NEXT:    add x12, sp, #16
; CHECK-SD-NEXT:    add x17, sp, #376
; CHECK-SD-NEXT:    mov v0.b[6], w6
; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x13]
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
; CHECK-SD-NEXT:    add x14, sp, #216
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x17]
; CHECK-SD-NEXT:    add x13, sp, #576
; CHECK-SD-NEXT:    add x11, sp, #224
; CHECK-SD-NEXT:    add x10, sp, #232
; CHECK-SD-NEXT:    add x15, sp, #240
; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x14]
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x13]
; CHECK-SD-NEXT:    add x13, sp, #384
; CHECK-SD-NEXT:    mov v0.b[7], w7
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x13]
; CHECK-SD-NEXT:    add x13, sp, #112
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
; CHECK-SD-NEXT:    add x13, sp, #32
; CHECK-SD-NEXT:    add x14, sp, #584
; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x14]
; CHECK-SD-NEXT:    add x11, sp, #312
; CHECK-SD-NEXT:    add x14, sp, #40
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #592
; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x12]
; CHECK-SD-NEXT:    add x12, sp, #24
; CHECK-SD-NEXT:    add x16, sp, #248
; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x10]
; CHECK-SD-NEXT:    ld1 { v4.b }[8], [x11]
; CHECK-SD-NEXT:    add x11, sp, #400
; CHECK-SD-NEXT:    add x9, sp, #256
; CHECK-SD-NEXT:    add x8, sp, #264
; CHECK-SD-NEXT:    add x10, sp, #72
; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x12]
; CHECK-SD-NEXT:    add x12, sp, #392
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x12]
; CHECK-SD-NEXT:    add x12, sp, #48
; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x15]
; CHECK-SD-NEXT:    add x15, sp, #120
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x13]
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x15]
; CHECK-SD-NEXT:    add x15, sp, #408
; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x11]
; CHECK-SD-NEXT:    add x13, sp, #56
; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x16]
; CHECK-SD-NEXT:    add x11, sp, #64
; CHECK-SD-NEXT:    add x16, sp, #616
; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x14]
; CHECK-SD-NEXT:    add x14, sp, #600
; CHECK-SD-NEXT:    ld1 { v4.b }[9], [x14]
; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x15]
; CHECK-SD-NEXT:    add x15, sp, #608
; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]
; CHECK-SD-NEXT:    add x9, sp, #488
; CHECK-SD-NEXT:    add x14, sp, #320
; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x12]
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x14]
; CHECK-SD-NEXT:    ld1 { v4.b }[10], [x15]
; CHECK-SD-NEXT:    add x14, sp, #624
; CHECK-SD-NEXT:    add x9, sp, #688
; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]
; CHECK-SD-NEXT:    add x8, sp, #432
; CHECK-SD-NEXT:    add x12, sp, #328
; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]
; CHECK-SD-NEXT:    add x13, sp, #416
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x12]
; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x13]
; CHECK-SD-NEXT:    ld1 { v4.b }[11], [x16]
; CHECK-SD-NEXT:    add x16, sp, #680
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x16]
; CHECK-SD-NEXT:    add x13, sp, #632
; CHECK-SD-NEXT:    add x12, sp, #504
; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x11]
; CHECK-SD-NEXT:    add x11, sp, #424
; CHECK-SD-NEXT:    add x15, sp, #128
; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[12], [x14]
; CHECK-SD-NEXT:    add x11, sp, #696
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x15]
; CHECK-SD-NEXT:    add x9, sp, #640
; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x10]
; CHECK-SD-NEXT:    add x10, sp, #496
; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v4.b }[13], [x13]
; CHECK-SD-NEXT:    add x10, sp, #440
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #512
; CHECK-SD-NEXT:    add x8, sp, #136
; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x10]
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x12]
; CHECK-SD-NEXT:    ld1 { v4.b }[14], [x9]
; CHECK-SD-NEXT:    add x9, sp, #448
; CHECK-SD-NEXT:    add x10, sp, #704
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]
; CHECK-SD-NEXT:    add x8, sp, #648
; CHECK-SD-NEXT:    add x10, sp, #520
; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x9]
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[15], [x8]
; CHECK-SD-NEXT:    add x8, sp, #456
; CHECK-SD-NEXT:    add x9, sp, #712
; CHECK-SD-NEXT:    sdot v19.2s, v3.8b, v2.8b
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
; CHECK-SD-NEXT:    addv s0, v17.4s
; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
; CHECK-SD-NEXT:    addp v1.2s, v19.2s, v19.2s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v4.16b
; CHECK-SD-NEXT:    sdot v18.2s, v7.8b, v6.8b
; CHECK-SD-NEXT:    fmov w9, s1
; CHECK-SD-NEXT:    addv s2, v16.4s
; CHECK-SD-NEXT:    addp v3.2s, v18.2s, v18.2s
; CHECK-SD-NEXT:    add w8, w8, w9
; CHECK-SD-NEXT:    fmov w10, s2
; CHECK-SD-NEXT:    fmov w11, s3
; CHECK-SD-NEXT:    add w9, w10, w11
; CHECK-SD-NEXT:    add w0, w8, w9
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v24i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    ldr w8, [sp, #80]
; CHECK-GI-NEXT:    ldr w9, [sp, #88]
; CHECK-GI-NEXT:    fmov s1, w0
; CHECK-GI-NEXT:    ldr w11, [sp, #336]
; CHECK-GI-NEXT:    ldr w10, [sp, #280]
; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
; CHECK-GI-NEXT:    fmov s0, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #96]
; CHECK-GI-NEXT:    ldr w12, [sp, #152]
; CHECK-GI-NEXT:    mov v1.b[1], w1
; CHECK-GI-NEXT:    fmov s4, w11
; CHECK-GI-NEXT:    ldr w11, [sp, #584]
; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v0.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #272]
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #144]
; CHECK-GI-NEXT:    mov v1.b[2], w2
; CHECK-GI-NEXT:    mov v0.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #528]
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    mov v2.b[1], w10
; CHECK-GI-NEXT:    ldr w9, [sp, #344]
; CHECK-GI-NEXT:    ldr w10, [sp, #536]
; CHECK-GI-NEXT:    fmov s5, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #288]
; CHECK-GI-NEXT:    mov v1.b[3], w3
; CHECK-GI-NEXT:    mov v3.b[1], w12
; CHECK-GI-NEXT:    mov v4.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #160]
; CHECK-GI-NEXT:    mov v5.b[1], w10
; CHECK-GI-NEXT:    mov v2.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #104]
; CHECK-GI-NEXT:    ldr w10, [sp, #352]
; CHECK-GI-NEXT:    mov v1.b[4], w4
; CHECK-GI-NEXT:    mov v3.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #544]
; CHECK-GI-NEXT:    mov v0.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #296]
; CHECK-GI-NEXT:    mov v4.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #360]
; CHECK-GI-NEXT:    mov v5.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #168]
; CHECK-GI-NEXT:    mov v2.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #112]
; CHECK-GI-NEXT:    mov v1.b[5], w5
; CHECK-GI-NEXT:    mov v3.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #552]
; CHECK-GI-NEXT:    mov v0.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #304]
; CHECK-GI-NEXT:    mov v4.b[3], w10
; CHECK-GI-NEXT:    mov v5.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #176]
; CHECK-GI-NEXT:    ldr w10, [sp, #368]
; CHECK-GI-NEXT:    mov v2.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #120]
; CHECK-GI-NEXT:    mov v1.b[6], w6
; CHECK-GI-NEXT:    mov v3.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #560]
; CHECK-GI-NEXT:    mov v0.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #312]
; CHECK-GI-NEXT:    mov v4.b[4], w10
; CHECK-GI-NEXT:    mov v5.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #184]
; CHECK-GI-NEXT:    ldr w10, [sp, #376]
; CHECK-GI-NEXT:    mov v2.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #128]
; CHECK-GI-NEXT:    mov v1.b[7], w7
; CHECK-GI-NEXT:    mov v3.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #568]
; CHECK-GI-NEXT:    mov v0.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #320]
; CHECK-GI-NEXT:    mov v4.b[5], w10
; CHECK-GI-NEXT:    mov v5.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #192]
; CHECK-GI-NEXT:    ldr w10, [sp, #384]
; CHECK-GI-NEXT:    mov v2.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #136]
; CHECK-GI-NEXT:    mov v3.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #576]
; CHECK-GI-NEXT:    mov v0.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #328]
; CHECK-GI-NEXT:    mov v4.b[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #200]
; CHECK-GI-NEXT:    mov v5.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #392]
; CHECK-GI-NEXT:    mov v2.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #464]
; CHECK-GI-NEXT:    mov v3.b[7], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #16]
; CHECK-GI-NEXT:    fmov s6, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #208]
; CHECK-GI-NEXT:    mov v4.b[7], w9
; CHECK-GI-NEXT:    mov v1.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #656]
; CHECK-GI-NEXT:    ldr w9, [sp, #472]
; CHECK-GI-NEXT:    mov v5.b[7], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #400]
; CHECK-GI-NEXT:    fmov d0, d0
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    mov v6.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #592]
; CHECK-GI-NEXT:    mov v3.b[8], w8
; CHECK-GI-NEXT:    ldr w10, [sp, #664]
; CHECK-GI-NEXT:    ldr w8, [sp, #24]
; CHECK-GI-NEXT:    mov v4.b[8], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #216]
; CHECK-GI-NEXT:    fmov d2, d2
; CHECK-GI-NEXT:    mov v5.b[8], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #480]
; CHECK-GI-NEXT:    mov v7.b[1], w10
; CHECK-GI-NEXT:    mov v1.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #408]
; CHECK-GI-NEXT:    ldr w10, [sp, #600]
; CHECK-GI-NEXT:    mov v3.b[9], w11
; CHECK-GI-NEXT:    mov v6.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #672]
; CHECK-GI-NEXT:    ldr w11, [sp, #32]
; CHECK-GI-NEXT:    mov v4.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #224]
; CHECK-GI-NEXT:    mov v5.b[9], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #488]
; CHECK-GI-NEXT:    mov v7.b[2], w9
; CHECK-GI-NEXT:    mov v1.b[10], w11
; CHECK-GI-NEXT:    ldr w9, [sp, #416]
; CHECK-GI-NEXT:    ldr w11, [sp, #608]
; CHECK-GI-NEXT:    mov v3.b[10], w8
; CHECK-GI-NEXT:    mov v6.b[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #680]
; CHECK-GI-NEXT:    ldr w8, [sp, #40]
; CHECK-GI-NEXT:    mov v4.b[10], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #232]
; CHECK-GI-NEXT:    mov v5.b[10], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #496]
; CHECK-GI-NEXT:    mov v7.b[3], w10
; CHECK-GI-NEXT:    mov v1.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #424]
; CHECK-GI-NEXT:    ldr w10, [sp, #616]
; CHECK-GI-NEXT:    mov v3.b[11], w9
; CHECK-GI-NEXT:    mov v6.b[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #688]
; CHECK-GI-NEXT:    ldr w9, [sp, #48]
; CHECK-GI-NEXT:    mov v4.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #240]
; CHECK-GI-NEXT:    mov v5.b[11], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #504]
; CHECK-GI-NEXT:    mov v7.b[4], w11
; CHECK-GI-NEXT:    mov v1.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #432]
; CHECK-GI-NEXT:    ldr w11, [sp, #624]
; CHECK-GI-NEXT:    mov v3.b[12], w8
; CHECK-GI-NEXT:    mov v6.b[5], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #696]
; CHECK-GI-NEXT:    ldr w8, [sp, #56]
; CHECK-GI-NEXT:    mov v4.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #248]
; CHECK-GI-NEXT:    mov v5.b[12], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #512]
; CHECK-GI-NEXT:    mov v7.b[5], w10
; CHECK-GI-NEXT:    mov v1.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #440]
; CHECK-GI-NEXT:    ldr w10, [sp, #632]
; CHECK-GI-NEXT:    mov v3.b[13], w9
; CHECK-GI-NEXT:    mov v6.b[6], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #704]
; CHECK-GI-NEXT:    ldr w9, [sp, #64]
; CHECK-GI-NEXT:    mov v4.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #256]
; CHECK-GI-NEXT:    mov v5.b[13], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #520]
; CHECK-GI-NEXT:    mov v7.b[6], w11
; CHECK-GI-NEXT:    mov v1.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #448]
; CHECK-GI-NEXT:    ldr w11, [sp, #640]
; CHECK-GI-NEXT:    mov v3.b[14], w8
; CHECK-GI-NEXT:    mov v6.b[7], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #712]
; CHECK-GI-NEXT:    ldr w8, [sp, #72]
; CHECK-GI-NEXT:    mov v4.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #264]
; CHECK-GI-NEXT:    mov v5.b[14], w11
; CHECK-GI-NEXT:    mov v7.b[7], w10
; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v2.16b
; CHECK-GI-NEXT:    mov v1.b[15], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #456]
; CHECK-GI-NEXT:    mov v3.b[15], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #648]
; CHECK-GI-NEXT:    fmov d6, d6
; CHECK-GI-NEXT:    mov v4.b[15], w8
; CHECK-GI-NEXT:    mov v5.b[15], w9
; CHECK-GI-NEXT:    fmov d7, d7
; CHECK-GI-NEXT:    sdot v17.4s, v1.16b, v3.16b
; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v5.16b
; CHECK-GI-NEXT:    sdot v16.4s, v6.16b, v7.16b
; CHECK-GI-NEXT:    add v0.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v1.4s, v19.4s, v16.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <24 x i8> %a to <24 x i32>
  %bz = sext <24 x i8> %b to <24 x i32>
  %m1 = mul nuw nsw <24 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
  %cz = sext <24 x i8> %c to <24 x i32>
  %dz = sext <24 x i8> %d to <24 x i32>
  %m2 = mul nuw nsw <24 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v24i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    ldr b1, [sp, #336]
; CHECK-SD-NEXT:    add x8, sp, #344
; CHECK-SD-NEXT:    add x9, sp, #400
; CHECK-SD-NEXT:    ldr b2, [sp, #80]
; CHECK-SD-NEXT:    ldr b3, [sp, #464]
; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #352
; CHECK-SD-NEXT:    add x10, sp, #408
; CHECK-SD-NEXT:    mov v0.b[1], w1
; CHECK-SD-NEXT:    add x11, sp, #472
; CHECK-SD-NEXT:    add x12, sp, #480
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #416
; CHECK-SD-NEXT:    add x13, sp, #488
; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #360
; CHECK-SD-NEXT:    add x14, sp, #496
; CHECK-SD-NEXT:    movi v4.16b, #1
; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
; CHECK-SD-NEXT:    mov v0.b[2], w2
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
; CHECK-SD-NEXT:    add x12, sp, #424
; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #368
; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v16.8b, #1
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x13]
; CHECK-SD-NEXT:    add x13, sp, #432
; CHECK-SD-NEXT:    mov v0.b[3], w3
; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #376
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x14]
; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #384
; CHECK-SD-NEXT:    mov v0.b[4], w4
; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
; CHECK-SD-NEXT:    add x8, sp, #392
; CHECK-SD-NEXT:    mov v0.b[5], w5
; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
; CHECK-SD-NEXT:    add x8, sp, #16
; CHECK-SD-NEXT:    mov v0.b[6], w6
; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x9]
; CHECK-SD-NEXT:    add x9, sp, #88
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x10]
; CHECK-SD-NEXT:    add x10, sp, #96
; CHECK-SD-NEXT:    mov v0.b[7], w7
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #56
; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]
; CHECK-SD-NEXT:    add x11, sp, #104
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #72
; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x8]
; CHECK-SD-NEXT:    add x8, sp, #24
; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x12]
; CHECK-SD-NEXT:    add x12, sp, #112
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
; CHECK-SD-NEXT:    add x12, sp, #440
; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x8]
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x13]
; CHECK-SD-NEXT:    add x13, sp, #504
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
; CHECK-SD-NEXT:    add x13, sp, #512
; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
; CHECK-SD-NEXT:    add x8, sp, #48
; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x12]
; CHECK-SD-NEXT:    add x12, sp, #448
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x13]
; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]
; CHECK-SD-NEXT:    add x9, sp, #64
; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x12]
; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]
; CHECK-SD-NEXT:    add x8, sp, #120
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #128
; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x10]
; CHECK-SD-NEXT:    add x10, sp, #136
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]
; CHECK-SD-NEXT:    add x8, sp, #456
; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]
; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x9]
; CHECK-SD-NEXT:    add x9, sp, #520
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b
; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x11]
; CHECK-SD-NEXT:    sdot v17.2s, v2.8b, v16.8b
; CHECK-SD-NEXT:    sdot v7.2s, v3.8b, v16.8b
; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v4.16b
; CHECK-SD-NEXT:    addv s3, v5.4s
; CHECK-SD-NEXT:    addp v1.2s, v17.2s, v17.2s
; CHECK-SD-NEXT:    addp v2.2s, v7.2s, v7.2s
; CHECK-SD-NEXT:    fmov w10, s3
; CHECK-SD-NEXT:    addv s0, v6.4s
; CHECK-SD-NEXT:    fmov w9, s1
; CHECK-SD-NEXT:    fmov w11, s2
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w8, w8, w9
; CHECK-SD-NEXT:    add w9, w10, w11
; CHECK-SD-NEXT:    add w0, w8, w9
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v24i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    ldr w9, [sp, #336]
; CHECK-GI-NEXT:    ldr w8, [sp, #344]
; CHECK-GI-NEXT:    fmov s0, w0
; CHECK-GI-NEXT:    ldr w10, [sp, #16]
; CHECK-GI-NEXT:    ldr w11, [sp, #88]
; CHECK-GI-NEXT:    movi v4.8b, #1
; CHECK-GI-NEXT:    fmov s1, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #464]
; CHECK-GI-NEXT:    ldr w12, [sp, #400]
; CHECK-GI-NEXT:    mov v0.b[1], w1
; CHECK-GI-NEXT:    movi v5.8b, #1
; CHECK-GI-NEXT:    movi v6.8b, #1
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #96]
; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v1.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #352]
; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v0.b[2], w2
; CHECK-GI-NEXT:    mov v5.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v6.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v1.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #360]
; CHECK-GI-NEXT:    mov v0.b[3], w3
; CHECK-GI-NEXT:    mov v1.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #368]
; CHECK-GI-NEXT:    mov v0.b[4], w4
; CHECK-GI-NEXT:    mov v1.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #376]
; CHECK-GI-NEXT:    mov v0.b[5], w5
; CHECK-GI-NEXT:    mov v1.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #384]
; CHECK-GI-NEXT:    mov v0.b[6], w6
; CHECK-GI-NEXT:    mov v1.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #392]
; CHECK-GI-NEXT:    mov v0.b[7], w7
; CHECK-GI-NEXT:    mov v1.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #80]
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #472]
; CHECK-GI-NEXT:    mov v0.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #408]
; CHECK-GI-NEXT:    mov v1.b[8], w12
; CHECK-GI-NEXT:    mov v2.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #24]
; CHECK-GI-NEXT:    mov v3.b[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #480]
; CHECK-GI-NEXT:    mov v0.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #32]
; CHECK-GI-NEXT:    mov v1.b[9], w10
; CHECK-GI-NEXT:    mov v2.b[2], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #416]
; CHECK-GI-NEXT:    mov v3.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #104]
; CHECK-GI-NEXT:    ldr w11, [sp, #488]
; CHECK-GI-NEXT:    mov v0.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #40]
; CHECK-GI-NEXT:    mov v1.b[10], w10
; CHECK-GI-NEXT:    mov v2.b[3], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #424]
; CHECK-GI-NEXT:    mov v3.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #112]
; CHECK-GI-NEXT:    ldr w11, [sp, #496]
; CHECK-GI-NEXT:    mov v0.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #48]
; CHECK-GI-NEXT:    mov v1.b[11], w10
; CHECK-GI-NEXT:    mov v2.b[4], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #432]
; CHECK-GI-NEXT:    mov v3.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #120]
; CHECK-GI-NEXT:    ldr w11, [sp, #504]
; CHECK-GI-NEXT:    mov v0.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #56]
; CHECK-GI-NEXT:    mov v1.b[12], w10
; CHECK-GI-NEXT:    mov v2.b[5], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #440]
; CHECK-GI-NEXT:    mov v3.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #128]
; CHECK-GI-NEXT:    ldr w11, [sp, #512]
; CHECK-GI-NEXT:    mov v0.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #64]
; CHECK-GI-NEXT:    mov v1.b[13], w10
; CHECK-GI-NEXT:    mov v2.b[6], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #448]
; CHECK-GI-NEXT:    mov v3.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #136]
; CHECK-GI-NEXT:    ldr w11, [sp, #520]
; CHECK-GI-NEXT:    mov v0.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #72]
; CHECK-GI-NEXT:    mov v1.b[14], w10
; CHECK-GI-NEXT:    mov v2.b[7], w11
; CHECK-GI-NEXT:    mov v3.b[7], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #456]
; CHECK-GI-NEXT:    mov v0.b[15], w8
; CHECK-GI-NEXT:    mov v1.b[15], w9
; CHECK-GI-NEXT:    fmov d2, d2
; CHECK-GI-NEXT:    fmov d3, d3
; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v5.16b
; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v6.16b
; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b
; CHECK-GI-NEXT:    sdot v17.4s, v3.16b, v4.16b
; CHECK-GI-NEXT:    add v1.4s, v18.4s, v7.4s
; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <24 x i8> %a to <24 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)
  %cz = sext <24 x i8> %c to <24 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v25i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldp q3, q0, [x1]
; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q1, [x0]
; CHECK-SD-NEXT:    umull2 v4.8h, v0.16b, v1.16b
; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    umull v1.8h, v3.8b, v2.8b
; CHECK-SD-NEXT:    umull2 v2.8h, v3.16b, v2.16b
; CHECK-SD-NEXT:    ushll v3.4s, v4.4h, #0
; CHECK-SD-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h
; CHECK-SD-NEXT:    uaddl v0.4s, v1.4h, v0.4h
; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
; CHECK-SD-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    uaddw v2.4s, v5.4s, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v25i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sub sp, sp, #112
; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp q2, q1, [x1]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill
; CHECK-GI-NEXT:    mov b6, v2.b[3]
; CHECK-GI-NEXT:    mov b7, v2.b[4]
; CHECK-GI-NEXT:    mov b16, v2.b[5]
; CHECK-GI-NEXT:    mov b19, v2.b[8]
; CHECK-GI-NEXT:    mov b4, v2.b[1]
; CHECK-GI-NEXT:    mov b5, v2.b[2]
; CHECK-GI-NEXT:    mov b17, v2.b[6]
; CHECK-GI-NEXT:    mov b18, v2.b[7]
; CHECK-GI-NEXT:    mov b20, v2.b[9]
; CHECK-GI-NEXT:    mov b21, v2.b[10]
; CHECK-GI-NEXT:    mov b22, v2.b[11]
; CHECK-GI-NEXT:    fmov w7, s2
; CHECK-GI-NEXT:    fmov w13, s6
; CHECK-GI-NEXT:    mov b6, v2.b[12]
; CHECK-GI-NEXT:    fmov w2, s7
; CHECK-GI-NEXT:    mov b7, v2.b[13]
; CHECK-GI-NEXT:    fmov w11, s16
; CHECK-GI-NEXT:    mov b16, v2.b[14]
; CHECK-GI-NEXT:    mov b23, v2.b[15]
; CHECK-GI-NEXT:    ldp q3, q2, [x0]
; CHECK-GI-NEXT:    fmov w26, s19
; CHECK-GI-NEXT:    fmov w19, s4
; CHECK-GI-NEXT:    stp s17, s18, [sp, #4] // 8-byte Folded Spill
; CHECK-GI-NEXT:    fmov w29, s5
; CHECK-GI-NEXT:    fmov w24, s20
; CHECK-GI-NEXT:    uxtb w8, w7
; CHECK-GI-NEXT:    mov b4, v3.b[2]
; CHECK-GI-NEXT:    mov b5, v3.b[1]
; CHECK-GI-NEXT:    uxtb w13, w13
; CHECK-GI-NEXT:    mov b17, v1.b[1]
; CHECK-GI-NEXT:    fmov w22, s21
; CHECK-GI-NEXT:    uxtb w26, w26
; CHECK-GI-NEXT:    mov b18, v1.b[2]
; CHECK-GI-NEXT:    fmov w18, s22
; CHECK-GI-NEXT:    uxtb w24, w24
; CHECK-GI-NEXT:    mov b19, v1.b[3]
; CHECK-GI-NEXT:    fmov w16, s6
; CHECK-GI-NEXT:    uxtb w19, w19
; CHECK-GI-NEXT:    mov b21, v1.b[4]
; CHECK-GI-NEXT:    fmov w15, s7
; CHECK-GI-NEXT:    uxtb w22, w22
; CHECK-GI-NEXT:    mov b7, v1.b[5]
; CHECK-GI-NEXT:    mov b6, v3.b[3]
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    fmov w12, s23
; CHECK-GI-NEXT:    mov b22, v1.b[6]
; CHECK-GI-NEXT:    mov b23, v1.b[7]
; CHECK-GI-NEXT:    mov b20, v3.b[4]
; CHECK-GI-NEXT:    fmov w28, s4
; CHECK-GI-NEXT:    fmov s4, w26
; CHECK-GI-NEXT:    fmov w14, s16
; CHECK-GI-NEXT:    fmov w27, s17
; CHECK-GI-NEXT:    fmov w5, s18
; CHECK-GI-NEXT:    uxtb w12, w12
; CHECK-GI-NEXT:    fmov w4, s19
; CHECK-GI-NEXT:    mov b19, v3.b[5]
; CHECK-GI-NEXT:    uxtb w28, w28
; CHECK-GI-NEXT:    fmov w3, s21
; CHECK-GI-NEXT:    mov b18, v3.b[6]
; CHECK-GI-NEXT:    uxtb w27, w27
; CHECK-GI-NEXT:    uxtb w5, w5
; CHECK-GI-NEXT:    fmov w1, s7
; CHECK-GI-NEXT:    mov b16, v3.b[7]
; CHECK-GI-NEXT:    fmov w0, s22
; CHECK-GI-NEXT:    mov b17, v3.b[8]
; CHECK-GI-NEXT:    fmov w17, s23
; CHECK-GI-NEXT:    mov b7, v3.b[9]
; CHECK-GI-NEXT:    fmov w30, s5
; CHECK-GI-NEXT:    mov b5, v3.b[10]
; CHECK-GI-NEXT:    mov b21, v3.b[11]
; CHECK-GI-NEXT:    fmov w25, s6
; CHECK-GI-NEXT:    mov b6, v3.b[12]
; CHECK-GI-NEXT:    fmov w23, s20
; CHECK-GI-NEXT:    mov b20, v3.b[13]
; CHECK-GI-NEXT:    mov b22, v3.b[14]
; CHECK-GI-NEXT:    fmov w6, s3
; CHECK-GI-NEXT:    mov b23, v3.b[15]
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    mov v4.h[1], w24
; CHECK-GI-NEXT:    fmov w21, s19
; CHECK-GI-NEXT:    mov b19, v2.b[1]
; CHECK-GI-NEXT:    fmov w9, s17
; CHECK-GI-NEXT:    fmov w24, s6
; CHECK-GI-NEXT:    fmov w7, s16
; CHECK-GI-NEXT:    mov b16, v2.b[2]
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    mov v3.h[1], w19
; CHECK-GI-NEXT:    uxtb w19, w29
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    fmov w29, s5
; CHECK-GI-NEXT:    mov v4.h[2], w22
; CHECK-GI-NEXT:    uxtb w22, w6
; CHECK-GI-NEXT:    fmov s5, w8
; CHECK-GI-NEXT:    fmov w10, s7
; CHECK-GI-NEXT:    fmov s7, w9
; CHECK-GI-NEXT:    fmov w9, s16
; CHECK-GI-NEXT:    fmov w20, s18
; CHECK-GI-NEXT:    uxtb w29, w29
; CHECK-GI-NEXT:    fmov s6, w22
; CHECK-GI-NEXT:    fmov w22, s2
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    mov v5.h[1], w27
; CHECK-GI-NEXT:    uxtb w27, w30
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov b18, v2.b[3]
; CHECK-GI-NEXT:    mov v3.h[2], w19
; CHECK-GI-NEXT:    uxtb w22, w22
; CHECK-GI-NEXT:    mov v6.h[1], w27
; CHECK-GI-NEXT:    fmov w27, s19
; CHECK-GI-NEXT:    mov v7.h[1], w10
; CHECK-GI-NEXT:    fmov w26, s21
; CHECK-GI-NEXT:    mov b17, v2.b[4]
; CHECK-GI-NEXT:    fmov s16, w22
; CHECK-GI-NEXT:    mov v5.h[2], w5
; CHECK-GI-NEXT:    uxtb w5, w25
; CHECK-GI-NEXT:    uxtb w27, w27
; CHECK-GI-NEXT:    fmov w10, s18
; CHECK-GI-NEXT:    mov v3.h[3], w13
; CHECK-GI-NEXT:    uxtb w13, w4
; CHECK-GI-NEXT:    mov v6.h[2], w28
; CHECK-GI-NEXT:    fmov w8, s20
; CHECK-GI-NEXT:    mov v16.h[1], w27
; CHECK-GI-NEXT:    mov v7.h[2], w29
; CHECK-GI-NEXT:    mov b20, v2.b[5]
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    fmov w22, s17
; CHECK-GI-NEXT:    mov v5.h[3], w13
; CHECK-GI-NEXT:    uxtb w13, w2
; CHECK-GI-NEXT:    mov v6.h[3], w5
; CHECK-GI-NEXT:    mov b21, v2.b[6]
; CHECK-GI-NEXT:    mov v16.h[2], w9
; CHECK-GI-NEXT:    uxtb w9, w18
; CHECK-GI-NEXT:    uxtb w18, w23
; CHECK-GI-NEXT:    mov v3.h[4], w13
; CHECK-GI-NEXT:    uxtb w13, w24
; CHECK-GI-NEXT:    fmov w27, s20
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v4.h[3], w9
; CHECK-GI-NEXT:    uxtb w9, w26
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v16.h[3], w10
; CHECK-GI-NEXT:    uxtb w10, w3
; CHECK-GI-NEXT:    mov v6.h[4], w18
; CHECK-GI-NEXT:    ldr w18, [sp, #4] // 4-byte Reload
; CHECK-GI-NEXT:    mov v7.h[3], w9
; CHECK-GI-NEXT:    uxtb w9, w16
; CHECK-GI-NEXT:    uxtb w16, w22
; CHECK-GI-NEXT:    mov v5.h[4], w10
; CHECK-GI-NEXT:    uxtb w10, w15
; CHECK-GI-NEXT:    uxtb w18, w18
; CHECK-GI-NEXT:    mov v4.h[4], w9
; CHECK-GI-NEXT:    uxtb w9, w21
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v16.h[4], w16
; CHECK-GI-NEXT:    mov v7.h[4], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #8] // 4-byte Reload
; CHECK-GI-NEXT:    mov v6.h[5], w9
; CHECK-GI-NEXT:    uxtb w9, w1
; CHECK-GI-NEXT:    mov v3.h[5], w11
; CHECK-GI-NEXT:    uxtb w11, w27
; CHECK-GI-NEXT:    fmov w19, s22
; CHECK-GI-NEXT:    fmov w28, s21
; CHECK-GI-NEXT:    uxtb w13, w13
; CHECK-GI-NEXT:    mov b17, v2.b[7]
; CHECK-GI-NEXT:    mov v5.h[5], w9
; CHECK-GI-NEXT:    uxtb w9, w0
; CHECK-GI-NEXT:    mov v4.h[5], w10
; CHECK-GI-NEXT:    uxtb w10, w20
; CHECK-GI-NEXT:    mov v7.h[5], w8
; CHECK-GI-NEXT:    mov v16.h[5], w11
; CHECK-GI-NEXT:    uxtb w8, w14
; CHECK-GI-NEXT:    uxtb w11, w28
; CHECK-GI-NEXT:    mov v6.h[6], w10
; CHECK-GI-NEXT:    uxtb w10, w19
; CHECK-GI-NEXT:    fmov w6, s23
; CHECK-GI-NEXT:    mov v5.h[6], w9
; CHECK-GI-NEXT:    fmov w9, s17
; CHECK-GI-NEXT:    mov v3.h[6], w18
; CHECK-GI-NEXT:    mov v4.h[6], w8
; CHECK-GI-NEXT:    uxtb w8, w7
; CHECK-GI-NEXT:    mov v7.h[6], w10
; CHECK-GI-NEXT:    mov v16.h[6], w11
; CHECK-GI-NEXT:    uxtb w10, w6
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v6.h[7], w8
; CHECK-GI-NEXT:    uxtb w8, w17
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov v3.h[7], w13
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v4.h[7], w12
; CHECK-GI-NEXT:    mov v5.h[7], w8
; CHECK-GI-NEXT:    mov v7.h[7], w10
; CHECK-GI-NEXT:    mov v16.h[7], w9
; CHECK-GI-NEXT:    umov w8, v1.b[8]
; CHECK-GI-NEXT:    umov w9, v2.b[8]
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mul v3.8h, v3.8h, v6.8h
; CHECK-GI-NEXT:    mul v2.8h, v4.8h, v7.8h
; CHECK-GI-NEXT:    mul v1.8h, v5.8h, v16.8h
; CHECK-GI-NEXT:    mul w15, w8, w9
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    umov w16, v3.h[0]
; CHECK-GI-NEXT:    umov w18, v3.h[4]
; CHECK-GI-NEXT:    umov w17, v3.h[1]
; CHECK-GI-NEXT:    umov w1, v2.h[0]
; CHECK-GI-NEXT:    umov w3, v2.h[4]
; CHECK-GI-NEXT:    umov w0, v3.h[5]
; CHECK-GI-NEXT:    umov w5, v1.h[0]
; CHECK-GI-NEXT:    umov w7, v1.h[4]
; CHECK-GI-NEXT:    umov w2, v2.h[1]
; CHECK-GI-NEXT:    umov w4, v2.h[5]
; CHECK-GI-NEXT:    umov w6, v1.h[1]
; CHECK-GI-NEXT:    umov w19, v1.h[5]
; CHECK-GI-NEXT:    umov w10, v3.h[2]
; CHECK-GI-NEXT:    umov w8, v3.h[3]
; CHECK-GI-NEXT:    umov w11, v3.h[6]
; CHECK-GI-NEXT:    umov w9, v3.h[7]
; CHECK-GI-NEXT:    fmov s3, w16
; CHECK-GI-NEXT:    fmov s4, w18
; CHECK-GI-NEXT:    fmov s5, w1
; CHECK-GI-NEXT:    fmov s6, w3
; CHECK-GI-NEXT:    fmov s7, w5
; CHECK-GI-NEXT:    fmov s16, w7
; CHECK-GI-NEXT:    fmov s17, w15
; CHECK-GI-NEXT:    umov w12, v2.h[2]
; CHECK-GI-NEXT:    umov w13, v2.h[6]
; CHECK-GI-NEXT:    umov w14, v1.h[2]
; CHECK-GI-NEXT:    umov w16, v1.h[6]
; CHECK-GI-NEXT:    mov v3.s[1], w17
; CHECK-GI-NEXT:    mov v4.s[1], w0
; CHECK-GI-NEXT:    mov v5.s[1], w2
; CHECK-GI-NEXT:    mov v6.s[1], w4
; CHECK-GI-NEXT:    mov v7.s[1], w6
; CHECK-GI-NEXT:    mov v16.s[1], w19
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v17.s[1], wzr
; CHECK-GI-NEXT:    umov w15, v2.h[3]
; CHECK-GI-NEXT:    umov w17, v2.h[7]
; CHECK-GI-NEXT:    umov w18, v1.h[3]
; CHECK-GI-NEXT:    umov w0, v1.h[7]
; CHECK-GI-NEXT:    mov v3.s[2], w10
; CHECK-GI-NEXT:    mov v4.s[2], w11
; CHECK-GI-NEXT:    mov v5.s[2], w12
; CHECK-GI-NEXT:    mov v6.s[2], w13
; CHECK-GI-NEXT:    mov v7.s[2], w14
; CHECK-GI-NEXT:    mov v16.s[2], w16
; CHECK-GI-NEXT:    mov v17.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[3], w8
; CHECK-GI-NEXT:    mov v4.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload
; CHECK-GI-NEXT:    mov v5.s[3], w15
; CHECK-GI-NEXT:    mov v6.s[3], w17
; CHECK-GI-NEXT:    mov v7.s[3], w18
; CHECK-GI-NEXT:    mov v16.s[3], w0
; CHECK-GI-NEXT:    mov v17.s[3], wzr
; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    add sp, sp, #112
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <25 x i8>, ptr %a
  %1 = zext <25 x i8> %0 to <25 x i32>
  %2 = load <25 x i8>, ptr %b
  %3 = zext <25 x i8> %2 to <25 x i32>
  %4 = mul nuw nsw <25 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v25i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldp q2, q1, [x0]
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ushll2 v3.8h, v1.16b, #0
; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
; CHECK-SD-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h
; CHECK-SD-NEXT:    uaddl v1.4s, v4.4h, v1.4h
; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
; CHECK-SD-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v25i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x19, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w19, -16
; CHECK-GI-NEXT:    ldp q2, q1, [x0]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    umov w15, v2.b[0]
; CHECK-GI-NEXT:    umov w17, v2.b[4]
; CHECK-GI-NEXT:    umov w0, v2.b[8]
; CHECK-GI-NEXT:    umov w2, v2.b[12]
; CHECK-GI-NEXT:    umov w4, v1.b[0]
; CHECK-GI-NEXT:    umov w6, v1.b[4]
; CHECK-GI-NEXT:    umov w19, v1.b[8]
; CHECK-GI-NEXT:    umov w16, v2.b[1]
; CHECK-GI-NEXT:    umov w18, v2.b[5]
; CHECK-GI-NEXT:    umov w1, v2.b[9]
; CHECK-GI-NEXT:    umov w3, v2.b[13]
; CHECK-GI-NEXT:    umov w5, v1.b[1]
; CHECK-GI-NEXT:    umov w7, v1.b[5]
; CHECK-GI-NEXT:    fmov s3, w15
; CHECK-GI-NEXT:    fmov s4, w17
; CHECK-GI-NEXT:    fmov s5, w0
; CHECK-GI-NEXT:    fmov s6, w2
; CHECK-GI-NEXT:    fmov s7, w4
; CHECK-GI-NEXT:    fmov s16, w6
; CHECK-GI-NEXT:    fmov s17, w19
; CHECK-GI-NEXT:    umov w10, v2.b[2]
; CHECK-GI-NEXT:    umov w11, v2.b[6]
; CHECK-GI-NEXT:    umov w12, v2.b[10]
; CHECK-GI-NEXT:    umov w13, v2.b[14]
; CHECK-GI-NEXT:    umov w14, v1.b[2]
; CHECK-GI-NEXT:    umov w15, v1.b[6]
; CHECK-GI-NEXT:    mov v3.s[1], w16
; CHECK-GI-NEXT:    mov v4.s[1], w18
; CHECK-GI-NEXT:    mov v5.s[1], w1
; CHECK-GI-NEXT:    mov v6.s[1], w3
; CHECK-GI-NEXT:    mov v7.s[1], w5
; CHECK-GI-NEXT:    mov v16.s[1], w7
; CHECK-GI-NEXT:    mov v17.s[1], wzr
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    umov w8, v2.b[3]
; CHECK-GI-NEXT:    umov w9, v2.b[7]
; CHECK-GI-NEXT:    umov w16, v2.b[11]
; CHECK-GI-NEXT:    umov w17, v2.b[15]
; CHECK-GI-NEXT:    umov w18, v1.b[3]
; CHECK-GI-NEXT:    umov w0, v1.b[7]
; CHECK-GI-NEXT:    mov v3.s[2], w10
; CHECK-GI-NEXT:    mov v4.s[2], w11
; CHECK-GI-NEXT:    mov v5.s[2], w12
; CHECK-GI-NEXT:    mov v6.s[2], w13
; CHECK-GI-NEXT:    mov v7.s[2], w14
; CHECK-GI-NEXT:    mov v16.s[2], w15
; CHECK-GI-NEXT:    mov v17.s[2], wzr
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[3], w8
; CHECK-GI-NEXT:    mov v4.s[3], w9
; CHECK-GI-NEXT:    mov v5.s[3], w16
; CHECK-GI-NEXT:    mov v6.s[3], w17
; CHECK-GI-NEXT:    mov v7.s[3], w18
; CHECK-GI-NEXT:    mov v16.s[3], w0
; CHECK-GI-NEXT:    mov v17.s[3], wzr
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ldr x19, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <25 x i8>, ptr %a1
  %1 = zext <25 x i8> %0 to <25 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v25i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldp q3, q0, [x1]
; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q1, [x0]
; CHECK-SD-NEXT:    smull2 v4.8h, v0.16b, v1.16b
; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    smull v1.8h, v3.8b, v2.8b
; CHECK-SD-NEXT:    smull2 v2.8h, v3.16b, v2.16b
; CHECK-SD-NEXT:    sshll v3.4s, v4.4h, #0
; CHECK-SD-NEXT:    saddl2 v4.4s, v1.8h, v0.8h
; CHECK-SD-NEXT:    saddl v0.4s, v1.4h, v0.4h
; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
; CHECK-SD-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v25i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sub sp, sp, #112
; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp q2, q1, [x1]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill
; CHECK-GI-NEXT:    mov b5, v2.b[2]
; CHECK-GI-NEXT:    mov b6, v2.b[3]
; CHECK-GI-NEXT:    mov b7, v2.b[4]
; CHECK-GI-NEXT:    mov b16, v2.b[5]
; CHECK-GI-NEXT:    mov b17, v2.b[6]
; CHECK-GI-NEXT:    mov b18, v2.b[7]
; CHECK-GI-NEXT:    mov b19, v2.b[8]
; CHECK-GI-NEXT:    mov b20, v2.b[9]
; CHECK-GI-NEXT:    mov b21, v2.b[15]
; CHECK-GI-NEXT:    mov b3, v2.b[1]
; CHECK-GI-NEXT:    fmov w19, s2
; CHECK-GI-NEXT:    mov b22, v1.b[6]
; CHECK-GI-NEXT:    fmov w6, s5
; CHECK-GI-NEXT:    mov b5, v2.b[10]
; CHECK-GI-NEXT:    fmov w14, s6
; CHECK-GI-NEXT:    mov b6, v2.b[11]
; CHECK-GI-NEXT:    fmov w2, s7
; CHECK-GI-NEXT:    stp s17, s18, [sp, #4] // 8-byte Folded Spill
; CHECK-GI-NEXT:    mov b7, v2.b[12]
; CHECK-GI-NEXT:    fmov w11, s16
; CHECK-GI-NEXT:    sxtb w28, w19
; CHECK-GI-NEXT:    mov b16, v2.b[13]
; CHECK-GI-NEXT:    mov b18, v1.b[1]
; CHECK-GI-NEXT:    sxtb w6, w6
; CHECK-GI-NEXT:    mov b17, v2.b[14]
; CHECK-GI-NEXT:    ldp q4, q2, [x0]
; CHECK-GI-NEXT:    fmov w25, s19
; CHECK-GI-NEXT:    fmov w24, s20
; CHECK-GI-NEXT:    fmov w22, s5
; CHECK-GI-NEXT:    mov b5, v1.b[2]
; CHECK-GI-NEXT:    fmov w0, s6
; CHECK-GI-NEXT:    sxtb w14, w14
; CHECK-GI-NEXT:    mov b20, v1.b[3]
; CHECK-GI-NEXT:    fmov w16, s7
; CHECK-GI-NEXT:    mov b7, v1.b[4]
; CHECK-GI-NEXT:    fmov w15, s16
; CHECK-GI-NEXT:    sxtb w25, w25
; CHECK-GI-NEXT:    sxtb w24, w24
; CHECK-GI-NEXT:    mov b16, v1.b[5]
; CHECK-GI-NEXT:    fmov w13, s21
; CHECK-GI-NEXT:    sxtb w22, w22
; CHECK-GI-NEXT:    mov b6, v4.b[2]
; CHECK-GI-NEXT:    fmov w26, s18
; CHECK-GI-NEXT:    sxtb w0, w0
; CHECK-GI-NEXT:    mov b21, v1.b[7]
; CHECK-GI-NEXT:    mov b18, v4.b[4]
; CHECK-GI-NEXT:    fmov w7, s3
; CHECK-GI-NEXT:    mov b3, v4.b[1]
; CHECK-GI-NEXT:    fmov w12, s17
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    mov b19, v4.b[3]
; CHECK-GI-NEXT:    fmov w4, s20
; CHECK-GI-NEXT:    fmov w3, s7
; CHECK-GI-NEXT:    sxtb w29, w7
; CHECK-GI-NEXT:    mov b17, v4.b[5]
; CHECK-GI-NEXT:    fmov w1, s16
; CHECK-GI-NEXT:    sxtb w5, w5
; CHECK-GI-NEXT:    mov b16, v4.b[6]
; CHECK-GI-NEXT:    fmov w18, s22
; CHECK-GI-NEXT:    mov b7, v4.b[7]
; CHECK-GI-NEXT:    fmov w17, s21
; CHECK-GI-NEXT:    mov b5, v4.b[8]
; CHECK-GI-NEXT:    mov b20, v4.b[9]
; CHECK-GI-NEXT:    fmov w27, s6
; CHECK-GI-NEXT:    mov b6, v4.b[10]
; CHECK-GI-NEXT:    mov b21, v4.b[11]
; CHECK-GI-NEXT:    fmov w21, s18
; CHECK-GI-NEXT:    mov b18, v4.b[12]
; CHECK-GI-NEXT:    mov b22, v4.b[13]
; CHECK-GI-NEXT:    mov b23, v4.b[14]
; CHECK-GI-NEXT:    fmov w10, s4
; CHECK-GI-NEXT:    sxtb w27, w27
; CHECK-GI-NEXT:    mov b24, v4.b[15]
; CHECK-GI-NEXT:    fmov s4, w25
; CHECK-GI-NEXT:    fmov w30, s3
; CHECK-GI-NEXT:    fmov s3, w28
; CHECK-GI-NEXT:    fmov w9, s5
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    fmov w7, s7
; CHECK-GI-NEXT:    mov b7, v2.b[1]
; CHECK-GI-NEXT:    mov v4.h[1], w24
; CHECK-GI-NEXT:    fmov w24, s1
; CHECK-GI-NEXT:    fmov w8, s20
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v3.h[1], w29
; CHECK-GI-NEXT:    fmov w29, s6
; CHECK-GI-NEXT:    fmov s6, w10
; CHECK-GI-NEXT:    fmov w10, s2
; CHECK-GI-NEXT:    fmov w19, s16
; CHECK-GI-NEXT:    sxtb w24, w24
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov b16, v2.b[3]
; CHECK-GI-NEXT:    sxtb w29, w29
; CHECK-GI-NEXT:    fmov w23, s19
; CHECK-GI-NEXT:    mov b19, v2.b[2]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    fmov s5, w24
; CHECK-GI-NEXT:    sxtb w24, w30
; CHECK-GI-NEXT:    mov v3.h[2], w6
; CHECK-GI-NEXT:    sxtb w6, w26
; CHECK-GI-NEXT:    fmov w28, s21
; CHECK-GI-NEXT:    sxtb w23, w23
; CHECK-GI-NEXT:    mov v6.h[1], w24
; CHECK-GI-NEXT:    fmov w24, s7
; CHECK-GI-NEXT:    fmov s7, w9
; CHECK-GI-NEXT:    fmov w9, s19
; CHECK-GI-NEXT:    mov v5.h[1], w6
; CHECK-GI-NEXT:    mov v4.h[2], w22
; CHECK-GI-NEXT:    fmov w20, s17
; CHECK-GI-NEXT:    mov b17, v2.b[4]
; CHECK-GI-NEXT:    sxtb w24, w24
; CHECK-GI-NEXT:    mov v3.h[3], w14
; CHECK-GI-NEXT:    sxtb w14, w2
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v7.h[1], w8
; CHECK-GI-NEXT:    fmov w8, s16
; CHECK-GI-NEXT:    fmov s16, w10
; CHECK-GI-NEXT:    mov v6.h[2], w27
; CHECK-GI-NEXT:    mov v5.h[2], w5
; CHECK-GI-NEXT:    fmov w25, s18
; CHECK-GI-NEXT:    mov v4.h[3], w0
; CHECK-GI-NEXT:    sxtb w0, w4
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov b18, v2.b[5]
; CHECK-GI-NEXT:    fmov w10, s17
; CHECK-GI-NEXT:    mov v16.h[1], w24
; CHECK-GI-NEXT:    mov v7.h[2], w29
; CHECK-GI-NEXT:    mov v3.h[4], w14
; CHECK-GI-NEXT:    sxtb w14, w25
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v6.h[3], w23
; CHECK-GI-NEXT:    mov v5.h[3], w0
; CHECK-GI-NEXT:    fmov w26, s22
; CHECK-GI-NEXT:    mov b19, v2.b[6]
; CHECK-GI-NEXT:    fmov w27, s18
; CHECK-GI-NEXT:    mov v16.h[2], w9
; CHECK-GI-NEXT:    sxtb w9, w28
; CHECK-GI-NEXT:    fmov w22, s23
; CHECK-GI-NEXT:    mov b17, v2.b[7]
; CHECK-GI-NEXT:    fmov w6, s24
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v7.h[3], w9
; CHECK-GI-NEXT:    sxtb w9, w11
; CHECK-GI-NEXT:    sxtb w11, w21
; CHECK-GI-NEXT:    fmov w24, s19
; CHECK-GI-NEXT:    mov v16.h[3], w8
; CHECK-GI-NEXT:    sxtb w8, w16
; CHECK-GI-NEXT:    sxtb w16, w3
; CHECK-GI-NEXT:    mov v6.h[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #4] // 4-byte Reload
; CHECK-GI-NEXT:    mov v3.h[5], w9
; CHECK-GI-NEXT:    sxtb w9, w15
; CHECK-GI-NEXT:    sxtb w15, w27
; CHECK-GI-NEXT:    mov v7.h[4], w14
; CHECK-GI-NEXT:    sxtb w14, w1
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v4.h[4], w8
; CHECK-GI-NEXT:    sxtb w8, w20
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v5.h[4], w16
; CHECK-GI-NEXT:    mov v16.h[4], w10
; CHECK-GI-NEXT:    sxtb w10, w26
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v6.h[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #8] // 4-byte Reload
; CHECK-GI-NEXT:    mov v7.h[5], w10
; CHECK-GI-NEXT:    sxtb w10, w12
; CHECK-GI-NEXT:    sxtb w12, w18
; CHECK-GI-NEXT:    mov v4.h[5], w9
; CHECK-GI-NEXT:    sxtb w9, w19
; CHECK-GI-NEXT:    mov v5.h[5], w14
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v16.h[5], w15
; CHECK-GI-NEXT:    mov v3.h[6], w11
; CHECK-GI-NEXT:    sxtb w11, w22
; CHECK-GI-NEXT:    mov v6.h[6], w9
; CHECK-GI-NEXT:    sxtb w9, w13
; CHECK-GI-NEXT:    sxtb w13, w24
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v7.h[6], w11
; CHECK-GI-NEXT:    fmov w11, s17
; CHECK-GI-NEXT:    mov v4.h[6], w10
; CHECK-GI-NEXT:    sxtb w10, w7
; CHECK-GI-NEXT:    mov v5.h[6], w12
; CHECK-GI-NEXT:    mov v16.h[6], w13
; CHECK-GI-NEXT:    mov v3.h[7], w8
; CHECK-GI-NEXT:    sxtb w8, w6
; CHECK-GI-NEXT:    smov w12, v1.b[8]
; CHECK-GI-NEXT:    mov v6.h[7], w10
; CHECK-GI-NEXT:    sxtb w10, w17
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v4.h[7], w9
; CHECK-GI-NEXT:    mov v7.h[7], w8
; CHECK-GI-NEXT:    smov w8, v2.b[8]
; CHECK-GI-NEXT:    mov v5.h[7], w10
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v16.h[7], w11
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    mul v3.8h, v3.8h, v6.8h
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mul v2.8h, v4.8h, v7.8h
; CHECK-GI-NEXT:    mul w16, w12, w8
; CHECK-GI-NEXT:    mul v1.8h, v5.8h, v16.8h
; CHECK-GI-NEXT:    smov w17, v3.h[0]
; CHECK-GI-NEXT:    smov w0, v3.h[4]
; CHECK-GI-NEXT:    sxth w16, w16
; CHECK-GI-NEXT:    smov w2, v2.h[0]
; CHECK-GI-NEXT:    smov w4, v2.h[4]
; CHECK-GI-NEXT:    smov w18, v3.h[1]
; CHECK-GI-NEXT:    smov w1, v3.h[5]
; CHECK-GI-NEXT:    smov w3, v2.h[1]
; CHECK-GI-NEXT:    smov w5, v2.h[5]
; CHECK-GI-NEXT:    smov w6, v1.h[0]
; CHECK-GI-NEXT:    smov w19, v1.h[4]
; CHECK-GI-NEXT:    smov w7, v1.h[1]
; CHECK-GI-NEXT:    smov w20, v1.h[5]
; CHECK-GI-NEXT:    smov w10, v3.h[2]
; CHECK-GI-NEXT:    smov w8, v3.h[3]
; CHECK-GI-NEXT:    smov w11, v3.h[6]
; CHECK-GI-NEXT:    smov w9, v3.h[7]
; CHECK-GI-NEXT:    fmov s3, w17
; CHECK-GI-NEXT:    fmov s4, w0
; CHECK-GI-NEXT:    fmov s5, w2
; CHECK-GI-NEXT:    fmov s6, w4
; CHECK-GI-NEXT:    fmov s7, w6
; CHECK-GI-NEXT:    fmov s16, w19
; CHECK-GI-NEXT:    fmov s17, w16
; CHECK-GI-NEXT:    smov w12, v2.h[2]
; CHECK-GI-NEXT:    smov w13, v2.h[6]
; CHECK-GI-NEXT:    smov w14, v1.h[2]
; CHECK-GI-NEXT:    smov w15, v1.h[6]
; CHECK-GI-NEXT:    mov v3.s[1], w18
; CHECK-GI-NEXT:    mov v4.s[1], w1
; CHECK-GI-NEXT:    mov v5.s[1], w3
; CHECK-GI-NEXT:    mov v6.s[1], w5
; CHECK-GI-NEXT:    mov v7.s[1], w7
; CHECK-GI-NEXT:    mov v16.s[1], w20
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v17.s[1], wzr
; CHECK-GI-NEXT:    smov w16, v2.h[3]
; CHECK-GI-NEXT:    smov w17, v2.h[7]
; CHECK-GI-NEXT:    smov w18, v1.h[3]
; CHECK-GI-NEXT:    smov w0, v1.h[7]
; CHECK-GI-NEXT:    mov v3.s[2], w10
; CHECK-GI-NEXT:    mov v4.s[2], w11
; CHECK-GI-NEXT:    mov v5.s[2], w12
; CHECK-GI-NEXT:    mov v6.s[2], w13
; CHECK-GI-NEXT:    mov v7.s[2], w14
; CHECK-GI-NEXT:    mov v16.s[2], w15
; CHECK-GI-NEXT:    mov v17.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[3], w8
; CHECK-GI-NEXT:    mov v4.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload
; CHECK-GI-NEXT:    mov v5.s[3], w16
; CHECK-GI-NEXT:    mov v6.s[3], w17
; CHECK-GI-NEXT:    mov v7.s[3], w18
; CHECK-GI-NEXT:    mov v16.s[3], w0
; CHECK-GI-NEXT:    mov v17.s[3], wzr
; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    add sp, sp, #112
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <25 x i8>, ptr %a
  %1 = sext <25 x i8> %0 to <25 x i32>
  %2 = load <25 x i8>, ptr %b
  %3 = sext <25 x i8> %2 to <25 x i32>
  %4 = mul nsw <25 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v25i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    ldr b0, [sp, #216]
; CHECK-SD-NEXT:    add x8, sp, #224
; CHECK-SD-NEXT:    ldr b1, [sp, #16]
; CHECK-SD-NEXT:    ldr b2, [sp, #280]
; CHECK-SD-NEXT:    add x9, sp, #240
; CHECK-SD-NEXT:    ldr b4, [sp, #80]
; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #24
; CHECK-SD-NEXT:    add x10, sp, #48
; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #232
; CHECK-SD-NEXT:    add x11, sp, #96
; CHECK-SD-NEXT:    ldr b5, [sp, #152]
; CHECK-SD-NEXT:    add x12, sp, #168
; CHECK-SD-NEXT:    ldr b6, [sp, #616]
; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    fmov s3, w0
; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #288
; CHECK-SD-NEXT:    ldr b7, [sp, #416]
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #40
; CHECK-SD-NEXT:    ldr b22, [sp, #744]
; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #248
; CHECK-SD-NEXT:    mov v3.b[1], w1
; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #88
; CHECK-SD-NEXT:    ldr b23, [sp, #544]
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #256
; CHECK-SD-NEXT:    ldr b19, [sp, #680]
; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
; CHECK-SD-NEXT:    add x9, sp, #296
; CHECK-SD-NEXT:    ldr b20, [sp, #480]
; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
; CHECK-SD-NEXT:    add x10, sp, #160
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
; CHECK-SD-NEXT:    add x11, sp, #304
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #56
; CHECK-SD-NEXT:    add x10, sp, #264
; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #64
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
; CHECK-SD-NEXT:    add x9, sp, #272
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x12]
; CHECK-SD-NEXT:    add x11, sp, #72
; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
; CHECK-SD-NEXT:    add x10, sp, #312
; CHECK-SD-NEXT:    mov v3.b[2], w2
; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
; CHECK-SD-NEXT:    add x8, sp, #104
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #112
; CHECK-SD-NEXT:    add x10, sp, #128
; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #320
; CHECK-SD-NEXT:    ldr b21, [sp, #552]
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
; CHECK-SD-NEXT:    add x9, sp, #176
; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #624
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #120
; CHECK-SD-NEXT:    add x9, sp, #328
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #184
; CHECK-SD-NEXT:    add x11, sp, #192
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #632
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
; CHECK-SD-NEXT:    add x9, sp, #640
; CHECK-SD-NEXT:    add x8, sp, #336
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]
; CHECK-SD-NEXT:    add x8, sp, #656
; CHECK-SD-NEXT:    smull v23.8h, v23.8b, v22.8b
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #648
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x10]
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #200
; CHECK-SD-NEXT:    add x10, sp, #136
; CHECK-SD-NEXT:    ldr b22, [sp, #352]
; CHECK-SD-NEXT:    add x12, sp, #360
; CHECK-SD-NEXT:    mov v3.b[3], w3
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #208
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x11]
; CHECK-SD-NEXT:    add x11, sp, #424
; CHECK-SD-NEXT:    add x10, sp, #488
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #560
; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #440
; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x11]
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #432
; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x12]
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
; CHECK-SD-NEXT:    add x11, sp, #496
; CHECK-SD-NEXT:    add x12, sp, #568
; CHECK-SD-NEXT:    add x13, sp, #368
; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x11]
; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x12]
; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x13]
; CHECK-SD-NEXT:    add x10, sp, #448
; CHECK-SD-NEXT:    mov v3.b[4], w4
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #688
; CHECK-SD-NEXT:    add x11, sp, #576
; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #696
; CHECK-SD-NEXT:    add x12, sp, #376
; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x12]
; CHECK-SD-NEXT:    add x11, sp, #512
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #504
; CHECK-SD-NEXT:    add x12, sp, #584
; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #704
; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
; CHECK-SD-NEXT:    add x13, sp, #384
; CHECK-SD-NEXT:    mov v3.b[5], w5
; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x12]
; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x13]
; CHECK-SD-NEXT:    add x10, sp, #456
; CHECK-SD-NEXT:    ldr b16, [sp, #344]
; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #712
; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x11]
; CHECK-SD-NEXT:    ldr b17, [sp, #144]
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
; CHECK-SD-NEXT:    add x10, sp, #520
; CHECK-SD-NEXT:    add x11, sp, #592
; CHECK-SD-NEXT:    add x12, sp, #392
; CHECK-SD-NEXT:    mov v3.b[6], w6
; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x9]
; CHECK-SD-NEXT:    add x9, sp, #720
; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x12]
; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
; CHECK-SD-NEXT:    add x8, sp, #664
; CHECK-SD-NEXT:    add x10, sp, #464
; CHECK-SD-NEXT:    add x11, sp, #528
; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
; CHECK-SD-NEXT:    add x9, sp, #728
; CHECK-SD-NEXT:    add x12, sp, #600
; CHECK-SD-NEXT:    add x13, sp, #400
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x13]
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #736
; CHECK-SD-NEXT:    mov v3.b[7], w7
; CHECK-SD-NEXT:    sshll v18.4s, v16.4h, #0
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    add x8, sp, #672
; CHECK-SD-NEXT:    add x10, sp, #472
; CHECK-SD-NEXT:    add x11, sp, #608
; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #536
; CHECK-SD-NEXT:    add x12, sp, #408
; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x12]
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
; CHECK-SD-NEXT:    sshll v23.4s, v23.4h, #0
; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    smull v1.8h, v4.8b, v2.8b
; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v5.8b
; CHECK-SD-NEXT:    smull v3.8h, v20.8b, v19.8b
; CHECK-SD-NEXT:    smull v4.8h, v22.8b, v21.8b
; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
; CHECK-SD-NEXT:    smull v5.8h, v7.8b, v6.8b
; CHECK-SD-NEXT:    mov v16.s[0], v23.s[0]
; CHECK-SD-NEXT:    saddl2 v6.4s, v2.8h, v1.8h
; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
; CHECK-SD-NEXT:    saddl2 v2.4s, v4.8h, v3.8h
; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v0.4h
; CHECK-SD-NEXT:    saddw v7.4s, v16.4s, v5.4h
; CHECK-SD-NEXT:    saddw2 v0.4s, v6.4s, v0.8h
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v5.8h
; CHECK-SD-NEXT:    add v3.4s, v3.4s, v7.4s
; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v25i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    lsl w8, w0, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #16]
; CHECK-GI-NEXT:    lsl w10, w1, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #24]
; CHECK-GI-NEXT:    lsl w12, w4, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #56]
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #64]
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    lsl w8, w11, #8
; CHECK-GI-NEXT:    lsl w11, w2, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #112]
; CHECK-GI-NEXT:    mov v2.h[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #32]
; CHECK-GI-NEXT:    sbfx w9, w11, #8, #8
; CHECK-GI-NEXT:    lsl w11, w3, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v4.h[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #152]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #40]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v4.h[2], w10
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #160]
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v2.h[3], w11
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #48]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v4.h[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #80]
; CHECK-GI-NEXT:    lsl w8, w11, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #168]
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v2.h[4], w12
; CHECK-GI-NEXT:    lsl w12, w5, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v3.h[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #88]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v4.h[4], w8
; CHECK-GI-NEXT:    lsl w8, w10, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #176]
; CHECK-GI-NEXT:    mov v2.h[5], w12
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    lsl w12, w6, #8
; CHECK-GI-NEXT:    fmov s6, w9
; CHECK-GI-NEXT:    sbfx w15, w8, #8, #8
; CHECK-GI-NEXT:    lsl w9, w10, #8
; CHECK-GI-NEXT:    mov v3.h[2], w11
; CHECK-GI-NEXT:    sbfx w11, w12, #8, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #96]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v4.h[5], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #224]
; CHECK-GI-NEXT:    mov v6.h[1], w15
; CHECK-GI-NEXT:    mov v2.h[6], w11
; CHECK-GI-NEXT:    lsl w15, w7, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #184]
; CHECK-GI-NEXT:    ldr w12, [sp, #104]
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #216]
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v2.h[7], w15
; CHECK-GI-NEXT:    lsl w15, w9, #8
; CHECK-GI-NEXT:    mov v4.h[6], w14
; CHECK-GI-NEXT:    mov v6.h[2], w10
; CHECK-GI-NEXT:    lsl w10, w13, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    sbfx w13, w15, #8, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #288]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    mov v3.h[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #192]
; CHECK-GI-NEXT:    fmov s5, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #232]
; CHECK-GI-NEXT:    ldr w9, [sp, #120]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v6.h[3], w12
; CHECK-GI-NEXT:    ldr w8, [sp, #72]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v5.h[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #280]
; CHECK-GI-NEXT:    sbfx w15, w11, #8, #8
; CHECK-GI-NEXT:    sbfx w12, w13, #8, #8
; CHECK-GI-NEXT:    lsl w13, w14, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #240]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v3.h[5], w15
; CHECK-GI-NEXT:    lsl w15, w16, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    mov v5.h[2], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #296]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #200]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v6.h[4], w15
; CHECK-GI-NEXT:    ldr w15, [sp, #304]
; CHECK-GI-NEXT:    ldr w11, [sp, #128]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v5.h[3], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #208]
; CHECK-GI-NEXT:    mov v7.h[1], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #248]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v6.h[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #272]
; CHECK-GI-NEXT:    mov v3.h[6], w10
; CHECK-GI-NEXT:    lsl w10, w14, #8
; CHECK-GI-NEXT:    sbfx w14, w15, #8, #8
; CHECK-GI-NEXT:    mov v7.h[2], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #256]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #320]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    mov v5.h[4], w13
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #312]
; CHECK-GI-NEXT:    mov v3.h[7], w10
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v4.h[7], w8
; CHECK-GI-NEXT:    mov v7.h[3], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #264]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w8, w9, #8, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #136]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    mov v5.h[5], w12
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mul v16.8h, v2.8h, v3.8h
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    sbfx w12, w14, #8, #8
; CHECK-GI-NEXT:    lsl w14, w15, #8
; CHECK-GI-NEXT:    mov v6.h[6], w11
; CHECK-GI-NEXT:    mov v7.h[4], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #328]
; CHECK-GI-NEXT:    ldr w10, [sp, #144]
; CHECK-GI-NEXT:    mov v5.h[6], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #336]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    smov w9, v16.h[0]
; CHECK-GI-NEXT:    smov w15, v16.h[4]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    smov w17, v16.h[5]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    sbfx w11, w13, #8, #8
; CHECK-GI-NEXT:    smov w13, v16.h[1]
; CHECK-GI-NEXT:    mov v7.h[5], w14
; CHECK-GI-NEXT:    mov v5.h[7], w8
; CHECK-GI-NEXT:    ldr w14, [sp, #344]
; CHECK-GI-NEXT:    ldr w8, [sp, #352]
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    fmov s3, w15
; CHECK-GI-NEXT:    lsl w9, w12, #8
; CHECK-GI-NEXT:    sbfx w12, w16, #8, #8
; CHECK-GI-NEXT:    sxtb w14, w14
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    mov v7.h[6], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #360]
; CHECK-GI-NEXT:    smov w15, v16.h[3]
; CHECK-GI-NEXT:    mov v2.s[1], w13
; CHECK-GI-NEXT:    smov w13, v16.h[2]
; CHECK-GI-NEXT:    mov v6.h[7], w12
; CHECK-GI-NEXT:    smov w12, v16.h[6]
; CHECK-GI-NEXT:    mov v3.s[1], w17
; CHECK-GI-NEXT:    mul v18.8h, v4.8h, v5.8h
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w16, w9, #8, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #368]
; CHECK-GI-NEXT:    mov v2.s[2], w13
; CHECK-GI-NEXT:    smov w13, v16.h[7]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v3.s[2], w12
; CHECK-GI-NEXT:    sbfx w12, w8, #8, #8
; CHECK-GI-NEXT:    mul w8, w10, w14
; CHECK-GI-NEXT:    smov w10, v18.h[0]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #376]
; CHECK-GI-NEXT:    fmov s16, w12
; CHECK-GI-NEXT:    smov w12, v18.h[1]
; CHECK-GI-NEXT:    mov v7.h[7], w16
; CHECK-GI-NEXT:    mov v2.s[3], w15
; CHECK-GI-NEXT:    smov w15, v18.h[4]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v3.s[3], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #416]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    fmov s4, w10
; CHECK-GI-NEXT:    mov v16.h[1], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #424]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #384]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    fmov s5, w15
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #432]
; CHECK-GI-NEXT:    mov v4.s[1], w12
; CHECK-GI-NEXT:    smov w12, v18.h[5]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v16.h[2], w9
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    fmov s17, w13
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mul v7.8h, v6.8h, v7.8h
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #392]
; CHECK-GI-NEXT:    ldr w13, [sp, #400]
; CHECK-GI-NEXT:    mov v5.s[1], w12
; CHECK-GI-NEXT:    smov w12, v18.h[2]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v17.h[1], w10
; CHECK-GI-NEXT:    mov v16.h[3], w14
; CHECK-GI-NEXT:    ldr w10, [sp, #440]
; CHECK-GI-NEXT:    smov w14, v18.h[6]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #456]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sxth w8, w8
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    mov v4.s[2], w12
; CHECK-GI-NEXT:    smov w12, v18.h[3]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v17.h[2], w15
; CHECK-GI-NEXT:    mov v16.h[4], w11
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    mov v5.s[2], w14
; CHECK-GI-NEXT:    smov w14, v18.h[7]
; CHECK-GI-NEXT:    ldr w15, [sp, #448]
; CHECK-GI-NEXT:    ldr w11, [sp, #408]
; CHECK-GI-NEXT:    mov v4.s[3], w12
; CHECK-GI-NEXT:    smov w12, v7.h[0]
; CHECK-GI-NEXT:    mov v17.h[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #480]
; CHECK-GI-NEXT:    mov v16.h[5], w9
; CHECK-GI-NEXT:    lsl w9, w13, #8
; CHECK-GI-NEXT:    lsl w13, w15, #8
; CHECK-GI-NEXT:    mov v5.s[3], w14
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    smov w14, v7.h[1]
; CHECK-GI-NEXT:    lsl w15, w16, #8
; CHECK-GI-NEXT:    fmov s6, w12
; CHECK-GI-NEXT:    ldr w12, [sp, #488]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v17.h[4], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #496]
; CHECK-GI-NEXT:    fmov s18, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #552]
; CHECK-GI-NEXT:    mov v6.s[1], w14
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #464]
; CHECK-GI-NEXT:    mov v16.h[6], w9
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v18.h[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #560]
; CHECK-GI-NEXT:    mov v17.h[5], w15
; CHECK-GI-NEXT:    sbfx w15, w10, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #512]
; CHECK-GI-NEXT:    fmov s19, w15
; CHECK-GI-NEXT:    ldr w15, [sp, #616]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v16.h[7], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #504]
; CHECK-GI-NEXT:    mov v18.h[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #568]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    mov v17.h[6], w14
; CHECK-GI-NEXT:    lsl w14, w15, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #576]
; CHECK-GI-NEXT:    mov v19.h[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #624]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w16, w11, #8, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    fmov s20, w14
; CHECK-GI-NEXT:    ldr w14, [sp, #680]
; CHECK-GI-NEXT:    mov v18.h[3], w16
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    mov v19.h[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #632]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    mov v20.h[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #688]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v18.h[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #584]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v19.h[3], w15
; CHECK-GI-NEXT:    fmov s21, w14
; CHECK-GI-NEXT:    ldr w15, [sp, #640]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v20.h[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #696]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #520]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v21.h[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #592]
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    mov v19.h[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #704]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v20.h[3], w15
; CHECK-GI-NEXT:    ldr w15, [sp, #648]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v21.h[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #600]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    mov v18.h[5], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #712]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v19.h[5], w12
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    ldr w12, [sp, #656]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v21.h[3], w10
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #608]
; CHECK-GI-NEXT:    mov v20.h[4], w15
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #528]
; CHECK-GI-NEXT:    ldr w15, [sp, #664]
; CHECK-GI-NEXT:    mov v19.h[6], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #720]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    mov v21.h[4], w11
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w16, w10, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    mov v20.h[5], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #728]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v19.h[7], w16
; CHECK-GI-NEXT:    ldr w9, [sp, #472]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v18.h[6], w14
; CHECK-GI-NEXT:    sbfx w14, w15, #8, #8
; CHECK-GI-NEXT:    mov v21.h[5], w13
; CHECK-GI-NEXT:    ldr w15, [sp, #672]
; CHECK-GI-NEXT:    ldr w11, [sp, #536]
; CHECK-GI-NEXT:    ldr w13, [sp, #736]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    mov v20.h[6], w14
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mul v19.8h, v16.8h, v19.8h
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v21.h[6], w12
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    smov w14, v7.h[2]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    smov w12, v7.h[4]
; CHECK-GI-NEXT:    mov v17.h[7], w9
; CHECK-GI-NEXT:    mov v20.h[7], w15
; CHECK-GI-NEXT:    smov w9, v7.h[5]
; CHECK-GI-NEXT:    mov v18.h[7], w11
; CHECK-GI-NEXT:    smov w11, v19.h[4]
; CHECK-GI-NEXT:    ldr w15, [sp, #744]
; CHECK-GI-NEXT:    mov v21.h[7], w13
; CHECK-GI-NEXT:    mov v6.s[2], w14
; CHECK-GI-NEXT:    smov w14, v19.h[0]
; CHECK-GI-NEXT:    fmov s16, w12
; CHECK-GI-NEXT:    smov w13, v19.h[5]
; CHECK-GI-NEXT:    smov w12, v19.h[1]
; CHECK-GI-NEXT:    mul v20.8h, v17.8h, v20.8h
; CHECK-GI-NEXT:    ldr w10, [sp, #544]
; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    mul v22.8h, v18.8h, v21.8h
; CHECK-GI-NEXT:    fmov s18, w11
; CHECK-GI-NEXT:    mov v16.s[1], w9
; CHECK-GI-NEXT:    fmov s17, w14
; CHECK-GI-NEXT:    smov w14, v7.h[6]
; CHECK-GI-NEXT:    smov w11, v19.h[2]
; CHECK-GI-NEXT:    smov w9, v7.h[3]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    fmov s21, w8
; CHECK-GI-NEXT:    mov v18.s[1], w13
; CHECK-GI-NEXT:    sxtb w13, w15
; CHECK-GI-NEXT:    smov w15, v20.h[0]
; CHECK-GI-NEXT:    mov v17.s[1], w12
; CHECK-GI-NEXT:    smov w8, v7.h[7]
; CHECK-GI-NEXT:    smov w12, v19.h[6]
; CHECK-GI-NEXT:    mov v16.s[2], w14
; CHECK-GI-NEXT:    smov w14, v20.h[1]
; CHECK-GI-NEXT:    mul w10, w10, w13
; CHECK-GI-NEXT:    smov w13, v20.h[4]
; CHECK-GI-NEXT:    smov w16, v20.h[5]
; CHECK-GI-NEXT:    mov v21.s[1], wzr
; CHECK-GI-NEXT:    fmov s7, w15
; CHECK-GI-NEXT:    smov w15, v20.h[2]
; CHECK-GI-NEXT:    mov v6.s[3], w9
; CHECK-GI-NEXT:    mov v17.s[2], w11
; CHECK-GI-NEXT:    smov w11, v22.h[0]
; CHECK-GI-NEXT:    sxth w10, w10
; CHECK-GI-NEXT:    mov v18.s[2], w12
; CHECK-GI-NEXT:    smov w12, v22.h[1]
; CHECK-GI-NEXT:    mov v16.s[3], w8
; CHECK-GI-NEXT:    mov v7.s[1], w14
; CHECK-GI-NEXT:    smov w14, v22.h[4]
; CHECK-GI-NEXT:    fmov s23, w13
; CHECK-GI-NEXT:    smov w13, v22.h[5]
; CHECK-GI-NEXT:    fmov s26, w10
; CHECK-GI-NEXT:    smov w10, v19.h[7]
; CHECK-GI-NEXT:    fmov s24, w11
; CHECK-GI-NEXT:    smov w11, v20.h[6]
; CHECK-GI-NEXT:    mov v21.s[2], wzr
; CHECK-GI-NEXT:    mov v23.s[1], w16
; CHECK-GI-NEXT:    add v4.4s, v6.4s, v16.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    fmov s25, w14
; CHECK-GI-NEXT:    smov w14, v22.h[2]
; CHECK-GI-NEXT:    mov v26.s[1], wzr
; CHECK-GI-NEXT:    mov v24.s[1], w12
; CHECK-GI-NEXT:    smov w12, v19.h[3]
; CHECK-GI-NEXT:    mov v7.s[2], w15
; CHECK-GI-NEXT:    smov w15, v20.h[3]
; CHECK-GI-NEXT:    mov v18.s[3], w10
; CHECK-GI-NEXT:    mov v21.s[3], wzr
; CHECK-GI-NEXT:    mov v25.s[1], w13
; CHECK-GI-NEXT:    smov w13, v22.h[6]
; CHECK-GI-NEXT:    mov v23.s[2], w11
; CHECK-GI-NEXT:    smov w11, v20.h[7]
; CHECK-GI-NEXT:    mov v26.s[2], wzr
; CHECK-GI-NEXT:    mov v24.s[2], w14
; CHECK-GI-NEXT:    smov w14, v22.h[3]
; CHECK-GI-NEXT:    mov v17.s[3], w12
; CHECK-GI-NEXT:    mov v7.s[3], w15
; CHECK-GI-NEXT:    add v1.4s, v21.4s, v1.4s
; CHECK-GI-NEXT:    mov v25.s[2], w13
; CHECK-GI-NEXT:    smov w13, v22.h[7]
; CHECK-GI-NEXT:    mov v23.s[3], w11
; CHECK-GI-NEXT:    mov v26.s[3], wzr
; CHECK-GI-NEXT:    mov v24.s[3], w14
; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v1.4s, v4.4s, v1.4s
; CHECK-GI-NEXT:    mov v25.s[3], w13
; CHECK-GI-NEXT:    add v6.4s, v7.4s, v23.4s
; CHECK-GI-NEXT:    add v0.4s, v26.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    add v7.4s, v24.4s, v25.4s
; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v7.4s, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <25 x i8> %a to <25 x i32>
  %bz = sext <25 x i8> %b to <25 x i32>
  %m1 = mul nuw nsw <25 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
  %cz = sext <25 x i8> %c to <25 x i32>
  %dz = sext <25 x i8> %d to <25 x i32>
  %m2 = mul nuw nsw <25 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v25i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    ldr b1, [sp, #80]
; CHECK-SD-NEXT:    add x10, sp, #88
; CHECK-SD-NEXT:    ldr b2, [sp, #16]
; CHECK-SD-NEXT:    add x9, sp, #96
; CHECK-SD-NEXT:    ldr b3, [sp, #480]
; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
; CHECK-SD-NEXT:    add x10, sp, #24
; CHECK-SD-NEXT:    ldr b4, [sp, #352]
; CHECK-SD-NEXT:    mov v0.b[1], w1
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x10]
; CHECK-SD-NEXT:    add x11, sp, #488
; CHECK-SD-NEXT:    add x10, sp, #360
; CHECK-SD-NEXT:    ldr b5, [sp, #416]
; CHECK-SD-NEXT:    add x8, sp, #104
; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #32
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
; CHECK-SD-NEXT:    add x11, sp, #424
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x10]
; CHECK-SD-NEXT:    mov v0.b[2], w2
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
; CHECK-SD-NEXT:    add x9, sp, #368
; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #40
; CHECK-SD-NEXT:    add x12, sp, #496
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]
; CHECK-SD-NEXT:    add x8, sp, #432
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
; CHECK-SD-NEXT:    add x13, sp, #48
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
; CHECK-SD-NEXT:    mov v0.b[3], w3
; CHECK-SD-NEXT:    add x10, sp, #112
; CHECK-SD-NEXT:    add x8, sp, #504
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x13]
; CHECK-SD-NEXT:    add x13, sp, #376
; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
; CHECK-SD-NEXT:    add x13, sp, #440
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
; CHECK-SD-NEXT:    add x11, sp, #120
; CHECK-SD-NEXT:    add x8, sp, #56
; CHECK-SD-NEXT:    mov v0.b[4], w4
; CHECK-SD-NEXT:    add x13, sp, #512
; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x11]
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #384
; CHECK-SD-NEXT:    add x11, sp, #448
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
; CHECK-SD-NEXT:    add x12, sp, #128
; CHECK-SD-NEXT:    add x10, sp, #64
; CHECK-SD-NEXT:    add x8, sp, #520
; CHECK-SD-NEXT:    mov v0.b[5], w5
; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
; CHECK-SD-NEXT:    add x10, sp, #392
; CHECK-SD-NEXT:    add x11, sp, #456
; CHECK-SD-NEXT:    ldr b6, [sp, #144]
; CHECK-SD-NEXT:    ldr b7, [sp, #544]
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
; CHECK-SD-NEXT:    add x9, sp, #136
; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
; CHECK-SD-NEXT:    mov v0.b[6], w6
; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
; CHECK-SD-NEXT:    add x8, sp, #528
; CHECK-SD-NEXT:    add x9, sp, #400
; CHECK-SD-NEXT:    add x10, sp, #464
; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    add x14, sp, #72
; CHECK-SD-NEXT:    mov v0.b[7], w7
; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
; CHECK-SD-NEXT:    add x8, sp, #536
; CHECK-SD-NEXT:    add x9, sp, #408
; CHECK-SD-NEXT:    add x10, sp, #472
; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x14]
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x10]
; CHECK-SD-NEXT:    mov v16.s[0], v6.s[0]
; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    mov v17.s[0], v7.s[0]
; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
; CHECK-SD-NEXT:    saddl v7.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    saddl2 v0.4s, v0.8h, v1.8h
; CHECK-SD-NEXT:    saddw v6.4s, v16.4s, v2.4h
; CHECK-SD-NEXT:    saddl v1.4s, v4.4h, v3.4h
; CHECK-SD-NEXT:    saddl2 v3.4s, v4.8h, v3.8h
; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v5.4h
; CHECK-SD-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
; CHECK-SD-NEXT:    add v6.4s, v7.4s, v6.4s
; CHECK-SD-NEXT:    saddw2 v2.4s, v3.4s, v5.8h
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-SD-NEXT:    add v0.4s, v6.4s, v0.4s
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v25i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    sxtb w8, w0
; CHECK-GI-NEXT:    sxtb w9, w1
; CHECK-GI-NEXT:    ldr w10, [sp, #48]
; CHECK-GI-NEXT:    sxtb w11, w6
; CHECK-GI-NEXT:    sxtb w12, w3
; CHECK-GI-NEXT:    sxtb w13, w7
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    sxtb w8, w4
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    ldr w14, [sp, #448]
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    sxtb w8, w2
; CHECK-GI-NEXT:    fmov s5, w10
; CHECK-GI-NEXT:    mov v2.s[1], w9
; CHECK-GI-NEXT:    sxtb w9, w5
; CHECK-GI-NEXT:    ldr w10, [sp, #80]
; CHECK-GI-NEXT:    mov v3.s[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #16]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[2], w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    ldr w8, [sp, #24]
; CHECK-GI-NEXT:    fmov s6, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #64]
; CHECK-GI-NEXT:    mov v3.s[2], w11
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    ldr w9, [sp, #56]
; CHECK-GI-NEXT:    ldr w11, [sp, #32]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v2.s[3], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #88]
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #120]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v3.s[3], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #112]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v5.s[1], w9
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    mov v6.s[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #128]
; CHECK-GI-NEXT:    mov v4.s[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #104]
; CHECK-GI-NEXT:    ldr w9, [sp, #40]
; CHECK-GI-NEXT:    fmov s7, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #96]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v5.s[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #136]
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v7.s[1], w8
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    ldr w8, [sp, #72]
; CHECK-GI-NEXT:    mov v6.s[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #144]
; CHECK-GI-NEXT:    mov v4.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #360]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    mov v7.s[2], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #352]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v6.s[3], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #384]
; CHECK-GI-NEXT:    fmov s16, w13
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    ldr w13, [sp, #416]
; CHECK-GI-NEXT:    mov v5.s[3], w8
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    ldr w8, [sp, #368]
; CHECK-GI-NEXT:    mov v7.s[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #392]
; CHECK-GI-NEXT:    fmov s17, w12
; CHECK-GI-NEXT:    fmov s18, w11
; CHECK-GI-NEXT:    ldr w12, [sp, #424]
; CHECK-GI-NEXT:    ldr w11, [sp, #456]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v17.s[1], w9
; CHECK-GI-NEXT:    mov v16.s[1], wzr
; CHECK-GI-NEXT:    mov v18.s[1], w10
; CHECK-GI-NEXT:    sxtb w10, w12
; CHECK-GI-NEXT:    sxtb w12, w14
; CHECK-GI-NEXT:    fmov s19, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #400]
; CHECK-GI-NEXT:    ldr w9, [sp, #376]
; CHECK-GI-NEXT:    fmov s20, w12
; CHECK-GI-NEXT:    ldr w12, [sp, #464]
; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    mov v17.s[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #488]
; CHECK-GI-NEXT:    mov v19.s[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #432]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v20.s[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #480]
; CHECK-GI-NEXT:    mov v18.s[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #512]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v16.s[2], wzr
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    mov v19.s[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #520]
; CHECK-GI-NEXT:    fmov s21, w11
; CHECK-GI-NEXT:    ldr w11, [sp, #544]
; CHECK-GI-NEXT:    mov v20.s[2], w12
; CHECK-GI-NEXT:    fmov s22, w13
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    ldr w13, [sp, #528]
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    ldr w12, [sp, #408]
; CHECK-GI-NEXT:    mov v17.s[3], w9
; CHECK-GI-NEXT:    mov v21.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #496]
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    fmov s23, w11
; CHECK-GI-NEXT:    mov v22.s[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #440]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    ldr w11, [sp, #472]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v16.s[3], wzr
; CHECK-GI-NEXT:    add v4.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    mov v23.s[1], wzr
; CHECK-GI-NEXT:    mov v21.s[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #504]
; CHECK-GI-NEXT:    mov v22.s[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #536]
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v18.s[3], w12
; CHECK-GI-NEXT:    mov v19.s[3], w10
; CHECK-GI-NEXT:    sxtb w9, w13
; CHECK-GI-NEXT:    mov v20.s[3], w11
; CHECK-GI-NEXT:    add v1.4s, v16.4s, v1.4s
; CHECK-GI-NEXT:    mov v23.s[2], wzr
; CHECK-GI-NEXT:    mov v21.s[3], w8
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    mov v22.s[3], w9
; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v1.4s, v4.4s, v1.4s
; CHECK-GI-NEXT:    add v6.4s, v19.4s, v20.4s
; CHECK-GI-NEXT:    mov v23.s[3], wzr
; CHECK-GI-NEXT:    add v7.4s, v21.4s, v22.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v0.4s, v23.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v7.4s, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <25 x i8> %a to <25 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
  %cz = sext <25 x i8> %c to <25 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v32i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q1, q3, [x0]
; CHECK-SD-NEXT:    ldp q2, q4, [x1]
; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v32i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q2, q3, [x0]
; CHECK-GI-NEXT:    ldp q4, q5, [x1]
; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <32 x i8>, ptr %a
  %1 = zext <32 x i8> %0 to <32 x i32>
  %2 = load <32 x i8>, ptr %b
  %3 = zext <32 x i8> %2 to <32 x i32>
  %4 = mul nuw nsw <32 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v32i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.16b, #1
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q3, [x0]
; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
; CHECK-SD-NEXT:    addv s0, v1.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v32i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.16b, #1
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q3, q4, [x0]
; CHECK-GI-NEXT:    udot v2.4s, v3.16b, v0.16b
; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <32 x i8>, ptr %a1
  %1 = zext <32 x i8> %0 to <32 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v32i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q1, q3, [x0]
; CHECK-SD-NEXT:    ldp q2, q4, [x1]
; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b
; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v32i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q2, q3, [x0]
; CHECK-GI-NEXT:    ldp q4, q5, [x1]
; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b
; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <32 x i8>, ptr %a
  %1 = sext <32 x i8> %0 to <32 x i32>
  %2 = load <32 x i8>, ptr %b
  %3 = sext <32 x i8> %2 to <32 x i32>
  %4 = mul nsw <32 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v32i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v17.4s, v1.16b, v3.16b
; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v7.16b
; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v2.16b
; CHECK-SD-NEXT:    sdot v16.4s, v4.16b, v6.16b
; CHECK-SD-NEXT:    add v0.4s, v17.4s, v16.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v32i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v2.16b
; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v3.16b
; CHECK-GI-NEXT:    sdot v17.4s, v5.16b, v7.16b
; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v6.16b
; CHECK-GI-NEXT:    add v0.4s, v16.4s, v18.4s
; CHECK-GI-NEXT:    add v1.4s, v19.4s, v17.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <32 x i8> %a to <32 x i32>
  %bz = sext <32 x i8> %b to <32 x i32>
  %m1 = mul nuw nsw <32 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
  %cz = sext <32 x i8> %c to <32 x i32>
  %dz = sext <32 x i8> %d to <32 x i32>
  %m2 = mul nuw nsw <32 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v32i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.16b, #1
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v6.4s, v1.16b, v2.16b
; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v2.16b
; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v2.16b
; CHECK-SD-NEXT:    sdot v3.4s, v4.16b, v2.16b
; CHECK-SD-NEXT:    add v0.4s, v6.4s, v3.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v32i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.16b, #1
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v6.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v3.4s, v0.16b, v2.16b
; CHECK-GI-NEXT:    sdot v6.4s, v5.16b, v2.16b
; CHECK-GI-NEXT:    sdot v7.4s, v1.16b, v2.16b
; CHECK-GI-NEXT:    sdot v16.4s, v4.16b, v2.16b
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v7.4s
; CHECK-GI-NEXT:    add v1.4s, v16.4s, v6.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <32 x i8> %a to <32 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)
  %cz = sext <32 x i8> %c to <32 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_usdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_usdot_v32i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q1, q3, [x0]
; CHECK-SD-NEXT:    ldp q2, q4, [x1]
; CHECK-SD-NEXT:    usdot v0.4s, v3.16b, v4.16b
; CHECK-SD-NEXT:    usdot v0.4s, v1.16b, v2.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v32i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldp q0, q1, [x1]
; CHECK-GI-NEXT:    ldp q2, q3, [x0]
; CHECK-GI-NEXT:    sshll v4.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    sshll v5.8h, v1.8b, #0
; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    ushll v6.8h, v2.8b, #0
; CHECK-GI-NEXT:    ushll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    ushll v7.8h, v3.8b, #0
; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v6.4h
; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v2.4h
; CHECK-GI-NEXT:    smull v18.4s, v5.4h, v7.4h
; CHECK-GI-NEXT:    smull v19.4s, v1.4h, v3.4h
; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v6.8h
; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v2.8h
; CHECK-GI-NEXT:    smlal2 v18.4s, v5.8h, v7.8h
; CHECK-GI-NEXT:    smlal2 v19.4s, v1.8h, v3.8h
; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <32 x i8>, ptr %a
  %1 = zext <32 x i8> %0 to <32 x i32>
  %2 = load <32 x i8>, ptr %b
  %3 = sext <32 x i8> %2 to <32 x i32>
  %4 = mul nsw <32 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_v32i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    usdot v17.4s, v1.16b, v3.16b
; CHECK-SD-NEXT:    usdot v16.4s, v5.16b, v7.16b
; CHECK-SD-NEXT:    usdot v17.4s, v0.16b, v2.16b
; CHECK-SD-NEXT:    usdot v16.4s, v4.16b, v6.16b
; CHECK-SD-NEXT:    add v0.4s, v17.4s, v16.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v32i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v16.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    ushll v17.8h, v1.8b, #0
; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    sshll v18.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    sshll v19.8h, v3.8b, #0
; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    ushll v20.8h, v4.8b, #0
; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #0
; CHECK-GI-NEXT:    ushll v21.8h, v5.8b, #0
; CHECK-GI-NEXT:    ushll2 v5.8h, v5.16b, #0
; CHECK-GI-NEXT:    sshll v22.8h, v6.8b, #0
; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #0
; CHECK-GI-NEXT:    sshll v23.8h, v7.8b, #0
; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #0
; CHECK-GI-NEXT:    smull v24.4s, v16.4h, v18.4h
; CHECK-GI-NEXT:    smull v25.4s, v0.4h, v2.4h
; CHECK-GI-NEXT:    smull v26.4s, v17.4h, v19.4h
; CHECK-GI-NEXT:    smull v27.4s, v1.4h, v3.4h
; CHECK-GI-NEXT:    smull v28.4s, v20.4h, v22.4h
; CHECK-GI-NEXT:    smull v29.4s, v4.4h, v6.4h
; CHECK-GI-NEXT:    smull v30.4s, v21.4h, v23.4h
; CHECK-GI-NEXT:    smull v31.4s, v5.4h, v7.4h
; CHECK-GI-NEXT:    smlal2 v24.4s, v16.8h, v18.8h
; CHECK-GI-NEXT:    smlal2 v25.4s, v0.8h, v2.8h
; CHECK-GI-NEXT:    smlal2 v26.4s, v17.8h, v19.8h
; CHECK-GI-NEXT:    smlal2 v27.4s, v1.8h, v3.8h
; CHECK-GI-NEXT:    smlal2 v28.4s, v20.8h, v22.8h
; CHECK-GI-NEXT:    smlal2 v29.4s, v4.8h, v6.8h
; CHECK-GI-NEXT:    smlal2 v30.4s, v21.8h, v23.8h
; CHECK-GI-NEXT:    smlal2 v31.4s, v5.8h, v7.8h
; CHECK-GI-NEXT:    add v0.4s, v24.4s, v25.4s
; CHECK-GI-NEXT:    add v1.4s, v26.4s, v27.4s
; CHECK-GI-NEXT:    add v2.4s, v28.4s, v29.4s
; CHECK-GI-NEXT:    add v3.4s, v30.4s, v31.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <32 x i8> %a to <32 x i32>
  %bz = sext <32 x i8> %b to <32 x i32>
  %m1 = mul nuw nsw <32 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
  %cz = zext <32 x i8> %c to <32 x i32>
  %dz = sext <32 x i8> %d to <32 x i32>
  %m2 = mul nuw nsw <32 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}


define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v33i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr b0, [x0, #32]
; CHECK-SD-NEXT:    ldr b1, [x1, #32]
; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q3, q4, [x1]
; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    ldp q1, q2, [x0]
; CHECK-SD-NEXT:    umull v5.8h, v4.8b, v2.8b
; CHECK-SD-NEXT:    umull v6.8h, v3.8b, v1.8b
; CHECK-SD-NEXT:    umull2 v2.8h, v4.16b, v2.16b
; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    umull2 v1.8h, v3.16b, v1.16b
; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
; CHECK-SD-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h
; CHECK-SD-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h
; CHECK-SD-NEXT:    uaddl v1.4s, v1.4h, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-SD-NEXT:    uaddw v2.4s, v7.4s, v6.4h
; CHECK-SD-NEXT:    uaddw v2.4s, v2.4s, v5.4h
; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v33i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sub sp, sp, #112
; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp q7, q16, [x1]
; CHECK-GI-NEXT:    movi d5, #0000000000000000
; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    movi d3, #0000000000000000
; CHECK-GI-NEXT:    movi d2, #0000000000000000
; CHECK-GI-NEXT:    mov b23, v7.b[7]
; CHECK-GI-NEXT:    mov b17, v7.b[1]
; CHECK-GI-NEXT:    fmov w11, s7
; CHECK-GI-NEXT:    mov b18, v7.b[2]
; CHECK-GI-NEXT:    mov b19, v7.b[3]
; CHECK-GI-NEXT:    mov b20, v7.b[4]
; CHECK-GI-NEXT:    mov b21, v7.b[5]
; CHECK-GI-NEXT:    mov b22, v7.b[6]
; CHECK-GI-NEXT:    mov b24, v7.b[8]
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    mov b25, v7.b[9]
; CHECK-GI-NEXT:    mov b26, v7.b[10]
; CHECK-GI-NEXT:    mov b27, v7.b[11]
; CHECK-GI-NEXT:    mov b28, v7.b[12]
; CHECK-GI-NEXT:    mov b29, v7.b[13]
; CHECK-GI-NEXT:    mov b30, v7.b[14]
; CHECK-GI-NEXT:    mov b7, v7.b[15]
; CHECK-GI-NEXT:    fmov w7, s23
; CHECK-GI-NEXT:    mov b23, v16.b[7]
; CHECK-GI-NEXT:    fmov w10, s17
; CHECK-GI-NEXT:    fmov w9, s18
; CHECK-GI-NEXT:    fmov w13, s19
; CHECK-GI-NEXT:    fmov w8, s24
; CHECK-GI-NEXT:    mov b17, v16.b[2]
; CHECK-GI-NEXT:    fmov w12, s20
; CHECK-GI-NEXT:    fmov w16, s25
; CHECK-GI-NEXT:    fmov w23, s21
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov b18, v16.b[1]
; CHECK-GI-NEXT:    stp s23, s7, [sp, #4] // 8-byte Folded Spill
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    fmov s7, w11
; CHECK-GI-NEXT:    fmov w5, s17
; CHECK-GI-NEXT:    fmov w27, s26
; CHECK-GI-NEXT:    mov b21, v16.b[5]
; CHECK-GI-NEXT:    fmov s17, w8
; CHECK-GI-NEXT:    uxtb w8, w12
; CHECK-GI-NEXT:    fmov w20, s22
; CHECK-GI-NEXT:    mov v7.h[1], w10
; CHECK-GI-NEXT:    uxtb w10, w16
; CHECK-GI-NEXT:    mov b19, v16.b[3]
; CHECK-GI-NEXT:    mov b22, v16.b[4]
; CHECK-GI-NEXT:    mov b20, v16.b[6]
; CHECK-GI-NEXT:    fmov w21, s27
; CHECK-GI-NEXT:    mov v17.h[1], w10
; CHECK-GI-NEXT:    fmov w24, s28
; CHECK-GI-NEXT:    mov b24, v16.b[8]
; CHECK-GI-NEXT:    fmov w22, s29
; CHECK-GI-NEXT:    mov b26, v16.b[9]
; CHECK-GI-NEXT:    fmov w4, s30
; CHECK-GI-NEXT:    uxtb w10, w21
; CHECK-GI-NEXT:    mov v7.h[2], w9
; CHECK-GI-NEXT:    uxtb w9, w13
; CHECK-GI-NEXT:    str s20, [sp] // 4-byte Spill
; CHECK-GI-NEXT:    mov b25, v16.b[10]
; CHECK-GI-NEXT:    fmov w25, s18
; CHECK-GI-NEXT:    uxtb w22, w22
; CHECK-GI-NEXT:    mov b27, v16.b[11]
; CHECK-GI-NEXT:    mov b28, v16.b[12]
; CHECK-GI-NEXT:    mov b29, v16.b[13]
; CHECK-GI-NEXT:    mov b30, v16.b[14]
; CHECK-GI-NEXT:    fmov w26, s16
; CHECK-GI-NEXT:    mov v7.h[3], w9
; CHECK-GI-NEXT:    uxtb w9, w27
; CHECK-GI-NEXT:    mov b31, v16.b[15]
; CHECK-GI-NEXT:    ldp q18, q16, [x0]
; CHECK-GI-NEXT:    fmov w2, s21
; CHECK-GI-NEXT:    uxtb w26, w26
; CHECK-GI-NEXT:    mov v17.h[2], w9
; CHECK-GI-NEXT:    fmov w14, s22
; CHECK-GI-NEXT:    fmov w3, s25
; CHECK-GI-NEXT:    fmov w15, s19
; CHECK-GI-NEXT:    fmov w19, s24
; CHECK-GI-NEXT:    mov v7.h[4], w8
; CHECK-GI-NEXT:    uxtb w8, w23
; CHECK-GI-NEXT:    mov b21, v18.b[2]
; CHECK-GI-NEXT:    mov b22, v18.b[1]
; CHECK-GI-NEXT:    mov b25, v18.b[5]
; CHECK-GI-NEXT:    mov b23, v18.b[6]
; CHECK-GI-NEXT:    uxtb w19, w19
; CHECK-GI-NEXT:    uxtb w3, w3
; CHECK-GI-NEXT:    mov v17.h[3], w10
; CHECK-GI-NEXT:    uxtb w10, w24
; CHECK-GI-NEXT:    uxtb w24, w7
; CHECK-GI-NEXT:    mov b19, v18.b[3]
; CHECK-GI-NEXT:    mov v7.h[5], w8
; CHECK-GI-NEXT:    uxtb w8, w20
; CHECK-GI-NEXT:    fmov w29, s21
; CHECK-GI-NEXT:    mov b21, v18.b[10]
; CHECK-GI-NEXT:    fmov w9, s22
; CHECK-GI-NEXT:    fmov w6, s26
; CHECK-GI-NEXT:    mov v17.h[4], w10
; CHECK-GI-NEXT:    uxtb w10, w25
; CHECK-GI-NEXT:    fmov w17, s27
; CHECK-GI-NEXT:    mov b26, v18.b[4]
; CHECK-GI-NEXT:    fmov w18, s28
; CHECK-GI-NEXT:    fmov w16, s29
; CHECK-GI-NEXT:    mov v7.h[6], w8
; CHECK-GI-NEXT:    fmov w8, s18
; CHECK-GI-NEXT:    mov b24, v18.b[7]
; CHECK-GI-NEXT:    fmov w30, s21
; CHECK-GI-NEXT:    mov b20, v18.b[8]
; CHECK-GI-NEXT:    mov b27, v18.b[9]
; CHECK-GI-NEXT:    uxtb w16, w16
; CHECK-GI-NEXT:    mov b28, v18.b[11]
; CHECK-GI-NEXT:    mov b29, v18.b[12]
; CHECK-GI-NEXT:    fmov w23, s25
; CHECK-GI-NEXT:    mov b25, v18.b[13]
; CHECK-GI-NEXT:    fmov w21, s23
; CHECK-GI-NEXT:    mov v7.h[7], w24
; CHECK-GI-NEXT:    uxtb w24, w8
; CHECK-GI-NEXT:    uxtb w8, w9
; CHECK-GI-NEXT:    uxtb w9, w29
; CHECK-GI-NEXT:    mov b23, v18.b[14]
; CHECK-GI-NEXT:    mov b22, v18.b[15]
; CHECK-GI-NEXT:    fmov s21, w24
; CHECK-GI-NEXT:    fmov s18, w26
; CHECK-GI-NEXT:    fmov w28, s19
; CHECK-GI-NEXT:    mov b19, v16.b[1]
; CHECK-GI-NEXT:    mov v17.h[5], w22
; CHECK-GI-NEXT:    fmov w7, s20
; CHECK-GI-NEXT:    fmov w11, s27
; CHECK-GI-NEXT:    fmov w27, s26
; CHECK-GI-NEXT:    mov b20, v16.b[2]
; CHECK-GI-NEXT:    mov v21.h[1], w8
; CHECK-GI-NEXT:    uxtb w8, w4
; CHECK-GI-NEXT:    mov v18.h[1], w10
; CHECK-GI-NEXT:    uxtb w10, w5
; CHECK-GI-NEXT:    uxtb w7, w7
; CHECK-GI-NEXT:    fmov w24, s23
; CHECK-GI-NEXT:    mov b23, v16.b[6]
; CHECK-GI-NEXT:    fmov w4, s22
; CHECK-GI-NEXT:    mov b22, v16.b[8]
; CHECK-GI-NEXT:    mov v17.h[6], w8
; CHECK-GI-NEXT:    fmov w8, s19
; CHECK-GI-NEXT:    fmov s19, w19
; CHECK-GI-NEXT:    mov v21.h[2], w9
; CHECK-GI-NEXT:    uxtb w9, w28
; CHECK-GI-NEXT:    mov v18.h[2], w10
; CHECK-GI-NEXT:    uxtb w10, w6
; CHECK-GI-NEXT:    mov b27, v16.b[9]
; CHECK-GI-NEXT:    fmov w20, s24
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    mov b24, v16.b[3]
; CHECK-GI-NEXT:    fmov w5, s20
; CHECK-GI-NEXT:    mov v19.h[1], w10
; CHECK-GI-NEXT:    fmov w10, s23
; CHECK-GI-NEXT:    fmov s23, w7
; CHECK-GI-NEXT:    mov v21.h[3], w9
; CHECK-GI-NEXT:    uxtb w9, w11
; CHECK-GI-NEXT:    uxtb w11, w27
; CHECK-GI-NEXT:    uxtb w27, w30
; CHECK-GI-NEXT:    uxtb w5, w5
; CHECK-GI-NEXT:    fmov w7, s22
; CHECK-GI-NEXT:    uxtb w10, w10
; CHECK-GI-NEXT:    mov v23.h[1], w9
; CHECK-GI-NEXT:    fmov w9, s16
; CHECK-GI-NEXT:    mov b20, v16.b[10]
; CHECK-GI-NEXT:    fmov w22, s28
; CHECK-GI-NEXT:    fmov w25, s25
; CHECK-GI-NEXT:    uxtb w7, w7
; CHECK-GI-NEXT:    mov v21.h[4], w11
; CHECK-GI-NEXT:    fmov w11, s27
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov b25, v16.b[5]
; CHECK-GI-NEXT:    fmov w29, s24
; CHECK-GI-NEXT:    fmov s22, w7
; CHECK-GI-NEXT:    mov v23.h[2], w27
; CHECK-GI-NEXT:    mov b24, v16.b[11]
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    fmov w27, s20
; CHECK-GI-NEXT:    fmov s20, w9
; CHECK-GI-NEXT:    fmov w26, s29
; CHECK-GI-NEXT:    mov b26, v16.b[4]
; CHECK-GI-NEXT:    mov v19.h[2], w3
; CHECK-GI-NEXT:    uxtb w3, w29
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v22.h[1], w11
; CHECK-GI-NEXT:    uxtb w11, w15
; CHECK-GI-NEXT:    uxtb w15, w22
; CHECK-GI-NEXT:    uxtb w22, w23
; CHECK-GI-NEXT:    mov v20.h[1], w8
; CHECK-GI-NEXT:    fmov w6, s25
; CHECK-GI-NEXT:    mov v18.h[3], w11
; CHECK-GI-NEXT:    uxtb w11, w27
; CHECK-GI-NEXT:    mov v23.h[3], w15
; CHECK-GI-NEXT:    uxtb w15, w17
; CHECK-GI-NEXT:    uxtb w17, w21
; CHECK-GI-NEXT:    mov b25, v16.b[12]
; CHECK-GI-NEXT:    fmov w28, s24
; CHECK-GI-NEXT:    mov v21.h[5], w22
; CHECK-GI-NEXT:    mov v22.h[2], w11
; CHECK-GI-NEXT:    uxtb w11, w14
; CHECK-GI-NEXT:    uxtb w14, w26
; CHECK-GI-NEXT:    mov v20.h[2], w5
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    fmov w19, s26
; CHECK-GI-NEXT:    mov v18.h[4], w11
; CHECK-GI-NEXT:    uxtb w11, w28
; CHECK-GI-NEXT:    mov v23.h[4], w14
; CHECK-GI-NEXT:    uxtb w14, w25
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov b26, v16.b[13]
; CHECK-GI-NEXT:    fmov w7, s25
; CHECK-GI-NEXT:    mov v19.h[3], w15
; CHECK-GI-NEXT:    uxtb w15, w18
; CHECK-GI-NEXT:    uxtb w18, w19
; CHECK-GI-NEXT:    mov v21.h[6], w17
; CHECK-GI-NEXT:    uxtb w17, w20
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v22.h[3], w11
; CHECK-GI-NEXT:    uxtb w11, w2
; CHECK-GI-NEXT:    mov v20.h[3], w3
; CHECK-GI-NEXT:    mov v23.h[5], w14
; CHECK-GI-NEXT:    uxtb w14, w24
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v18.h[5], w11
; CHECK-GI-NEXT:    uxtb w11, w7
; CHECK-GI-NEXT:    fmov w8, s26
; CHECK-GI-NEXT:    mov v19.h[4], w15
; CHECK-GI-NEXT:    ldr w15, [sp] // 4-byte Reload
; CHECK-GI-NEXT:    mov v21.h[7], w17
; CHECK-GI-NEXT:    uxtb w17, w6
; CHECK-GI-NEXT:    mov v22.h[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #8] // 4-byte Reload
; CHECK-GI-NEXT:    uxtb w8, w8
; CHECK-GI-NEXT:    uxtb w15, w15
; CHECK-GI-NEXT:    fmov w13, s30
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    mov v20.h[4], w18
; CHECK-GI-NEXT:    mov v23.h[6], w14
; CHECK-GI-NEXT:    mov v19.h[5], w16
; CHECK-GI-NEXT:    mov b27, v16.b[14]
; CHECK-GI-NEXT:    mul v24.8h, v7.8h, v21.8h
; CHECK-GI-NEXT:    mov v22.h[5], w8
; CHECK-GI-NEXT:    uxtb w8, w4
; CHECK-GI-NEXT:    mov b7, v16.b[7]
; CHECK-GI-NEXT:    mov b16, v16.b[15]
; CHECK-GI-NEXT:    fmov w12, s31
; CHECK-GI-NEXT:    mov v17.h[7], w11
; CHECK-GI-NEXT:    uxtb w11, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #4] // 4-byte Reload
; CHECK-GI-NEXT:    mov v20.h[5], w17
; CHECK-GI-NEXT:    mov v23.h[7], w8
; CHECK-GI-NEXT:    fmov w9, s27
; CHECK-GI-NEXT:    mov v18.h[6], w15
; CHECK-GI-NEXT:    uxtb w8, w12
; CHECK-GI-NEXT:    uxtb w13, w13
; CHECK-GI-NEXT:    mov v19.h[6], w11
; CHECK-GI-NEXT:    fmov w12, s16
; CHECK-GI-NEXT:    fmov w11, s7
; CHECK-GI-NEXT:    movi d4, #0000000000000000
; CHECK-GI-NEXT:    uxtb w9, w9
; CHECK-GI-NEXT:    mov v20.h[6], w10
; CHECK-GI-NEXT:    umov w10, v24.h[0]
; CHECK-GI-NEXT:    mul v21.8h, v17.8h, v23.8h
; CHECK-GI-NEXT:    mov v18.h[7], w13
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    uxtb w11, w11
; CHECK-GI-NEXT:    mov v19.h[7], w8
; CHECK-GI-NEXT:    uxtb w8, w12
; CHECK-GI-NEXT:    umov w12, v24.h[4]
; CHECK-GI-NEXT:    mov v22.h[6], w9
; CHECK-GI-NEXT:    umov w9, v24.h[1]
; CHECK-GI-NEXT:    mov v20.h[7], w11
; CHECK-GI-NEXT:    umov w11, v24.h[5]
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    ldrb w10, [x1, #32]
; CHECK-GI-NEXT:    umov w13, v21.h[0]
; CHECK-GI-NEXT:    umov w14, v21.h[1]
; CHECK-GI-NEXT:    umov w15, v21.h[4]
; CHECK-GI-NEXT:    mov v6.s[1], wzr
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    fmov s16, w12
; CHECK-GI-NEXT:    mov v22.h[7], w8
; CHECK-GI-NEXT:    umov w12, v24.h[6]
; CHECK-GI-NEXT:    umov w8, v24.h[2]
; CHECK-GI-NEXT:    mov v7.s[1], w9
; CHECK-GI-NEXT:    ldrb w9, [x0, #32]
; CHECK-GI-NEXT:    fmov s17, w13
; CHECK-GI-NEXT:    mul v23.8h, v18.8h, v20.8h
; CHECK-GI-NEXT:    umov w13, v24.h[7]
; CHECK-GI-NEXT:    mov v16.s[1], w11
; CHECK-GI-NEXT:    umov w11, v21.h[5]
; CHECK-GI-NEXT:    fmov s18, w15
; CHECK-GI-NEXT:    mul v19.8h, v19.8h, v22.8h
; CHECK-GI-NEXT:    umov w15, v21.h[6]
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v17.s[1], w14
; CHECK-GI-NEXT:    umov w14, v21.h[2]
; CHECK-GI-NEXT:    mov v7.s[2], w8
; CHECK-GI-NEXT:    mul w8, w10, w9
; CHECK-GI-NEXT:    umov w9, v23.h[0]
; CHECK-GI-NEXT:    umov w10, v23.h[1]
; CHECK-GI-NEXT:    mov v16.s[2], w12
; CHECK-GI-NEXT:    umov w12, v21.h[3]
; CHECK-GI-NEXT:    mov v18.s[1], w11
; CHECK-GI-NEXT:    umov w11, v23.h[4]
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v17.s[2], w14
; CHECK-GI-NEXT:    umov w14, v23.h[5]
; CHECK-GI-NEXT:    mov v4.s[1], wzr
; CHECK-GI-NEXT:    fmov s20, w9
; CHECK-GI-NEXT:    umov w9, v19.h[1]
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v16.s[3], w13
; CHECK-GI-NEXT:    umov w13, v19.h[0]
; CHECK-GI-NEXT:    mov v18.s[2], w15
; CHECK-GI-NEXT:    umov w15, v21.h[7]
; CHECK-GI-NEXT:    fmov s21, w11
; CHECK-GI-NEXT:    umov w11, v23.h[2]
; CHECK-GI-NEXT:    mov v17.s[3], w12
; CHECK-GI-NEXT:    umov w12, v19.h[4]
; CHECK-GI-NEXT:    mov v20.s[1], w10
; CHECK-GI-NEXT:    umov w10, v23.h[3]
; CHECK-GI-NEXT:    mov v6.s[2], wzr
; CHECK-GI-NEXT:    umov w16, v24.h[3]
; CHECK-GI-NEXT:    fmov s22, w13
; CHECK-GI-NEXT:    umov w13, v19.h[5]
; CHECK-GI-NEXT:    mov v21.s[1], w14
; CHECK-GI-NEXT:    umov w14, v23.h[6]
; CHECK-GI-NEXT:    mov v18.s[3], w15
; CHECK-GI-NEXT:    umov w15, v19.h[2]
; CHECK-GI-NEXT:    mov v20.s[2], w11
; CHECK-GI-NEXT:    umov w11, v19.h[6]
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v22.s[1], w9
; CHECK-GI-NEXT:    umov w9, v23.h[7]
; CHECK-GI-NEXT:    fmov s23, w12
; CHECK-GI-NEXT:    umov w12, v19.h[3]
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v21.s[2], w14
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], wzr
; CHECK-GI-NEXT:    mov v23.s[1], w13
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    mov v6.s[3], wzr
; CHECK-GI-NEXT:    mov v22.s[2], w15
; CHECK-GI-NEXT:    mov v7.s[3], w16
; CHECK-GI-NEXT:    mov v20.s[3], w10
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v21.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    mov v23.s[2], w11
; CHECK-GI-NEXT:    umov w11, v19.h[7]
; CHECK-GI-NEXT:    fmov s19, w8
; CHECK-GI-NEXT:    mov v22.s[3], w12
; CHECK-GI-NEXT:    mov v4.s[3], wzr
; CHECK-GI-NEXT:    add v5.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v6.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v7.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    mov v19.s[1], wzr
; CHECK-GI-NEXT:    add v16.4s, v20.4s, v21.4s
; CHECK-GI-NEXT:    mov v23.s[3], w11
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v5.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    mov v19.s[2], wzr
; CHECK-GI-NEXT:    add v17.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    mov v19.s[3], wzr
; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    add v0.4s, v19.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    add sp, sp, #112
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <33 x i8>, ptr %a
  %1 = zext <33 x i8> %0 to <33 x i32>
  %2 = load <33 x i8>, ptr %b
  %3 = zext <33 x i8> %2 to <33 x i32>
  %4 = mul nuw nsw <33 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v33i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr b1, [x0, #32]
; CHECK-SD-NEXT:    ldp q3, q2, [x0]
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
; CHECK-SD-NEXT:    ushll v5.8h, v3.8b, #0
; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
; CHECK-SD-NEXT:    ushll2 v3.8h, v3.16b, #0
; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-SD-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h
; CHECK-SD-NEXT:    mov v0.s[0], v1.s[0]
; CHECK-SD-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h
; CHECK-SD-NEXT:    uaddl v2.4s, v3.4h, v2.4h
; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
; CHECK-SD-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v4.4h
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v33i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    stp x20, x19, [sp, #-16]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    ldp q7, q19, [x0]
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    ldrb w10, [x0, #32]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi d3, #0000000000000000
; CHECK-GI-NEXT:    movi d2, #0000000000000000
; CHECK-GI-NEXT:    movi d5, #0000000000000000
; CHECK-GI-NEXT:    movi d4, #0000000000000000
; CHECK-GI-NEXT:    umov w15, v7.b[8]
; CHECK-GI-NEXT:    umov w2, v7.b[12]
; CHECK-GI-NEXT:    umov w16, v7.b[9]
; CHECK-GI-NEXT:    umov w4, v7.b[13]
; CHECK-GI-NEXT:    umov w8, v7.b[0]
; CHECK-GI-NEXT:    umov w12, v7.b[4]
; CHECK-GI-NEXT:    umov w9, v7.b[1]
; CHECK-GI-NEXT:    umov w14, v7.b[5]
; CHECK-GI-NEXT:    umov w3, v7.b[10]
; CHECK-GI-NEXT:    umov w5, v7.b[14]
; CHECK-GI-NEXT:    umov w11, v7.b[2]
; CHECK-GI-NEXT:    umov w17, v7.b[3]
; CHECK-GI-NEXT:    fmov s17, w15
; CHECK-GI-NEXT:    fmov s18, w2
; CHECK-GI-NEXT:    umov w13, v7.b[6]
; CHECK-GI-NEXT:    umov w18, v7.b[7]
; CHECK-GI-NEXT:    umov w0, v7.b[11]
; CHECK-GI-NEXT:    umov w1, v7.b[15]
; CHECK-GI-NEXT:    fmov s7, w8
; CHECK-GI-NEXT:    fmov s16, w12
; CHECK-GI-NEXT:    umov w2, v19.b[0]
; CHECK-GI-NEXT:    mov v17.s[1], w16
; CHECK-GI-NEXT:    mov v18.s[1], w4
; CHECK-GI-NEXT:    umov w4, v19.b[4]
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    umov w6, v19.b[1]
; CHECK-GI-NEXT:    umov w7, v19.b[5]
; CHECK-GI-NEXT:    mov v7.s[1], w9
; CHECK-GI-NEXT:    mov v16.s[1], w14
; CHECK-GI-NEXT:    umov w19, v19.b[9]
; CHECK-GI-NEXT:    umov w20, v19.b[13]
; CHECK-GI-NEXT:    umov w12, v19.b[2]
; CHECK-GI-NEXT:    umov w8, v19.b[3]
; CHECK-GI-NEXT:    mov v17.s[2], w3
; CHECK-GI-NEXT:    umov w3, v19.b[8]
; CHECK-GI-NEXT:    mov v18.s[2], w5
; CHECK-GI-NEXT:    umov w5, v19.b[12]
; CHECK-GI-NEXT:    umov w14, v19.b[6]
; CHECK-GI-NEXT:    umov w9, v19.b[7]
; CHECK-GI-NEXT:    mov v7.s[2], w11
; CHECK-GI-NEXT:    mov v16.s[2], w13
; CHECK-GI-NEXT:    umov w15, v19.b[10]
; CHECK-GI-NEXT:    umov w11, v19.b[11]
; CHECK-GI-NEXT:    umov w16, v19.b[14]
; CHECK-GI-NEXT:    umov w13, v19.b[15]
; CHECK-GI-NEXT:    fmov s19, w2
; CHECK-GI-NEXT:    fmov s20, w4
; CHECK-GI-NEXT:    fmov s21, w3
; CHECK-GI-NEXT:    fmov s22, w5
; CHECK-GI-NEXT:    fmov s23, w10
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v19.s[1], w6
; CHECK-GI-NEXT:    mov v20.s[1], w7
; CHECK-GI-NEXT:    mov v21.s[1], w19
; CHECK-GI-NEXT:    mov v22.s[1], w20
; CHECK-GI-NEXT:    mov v23.s[1], wzr
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    mov v4.s[1], wzr
; CHECK-GI-NEXT:    mov v6.s[1], wzr
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v19.s[2], w12
; CHECK-GI-NEXT:    mov v20.s[2], w14
; CHECK-GI-NEXT:    mov v21.s[2], w15
; CHECK-GI-NEXT:    mov v22.s[2], w16
; CHECK-GI-NEXT:    mov v23.s[2], wzr
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], wzr
; CHECK-GI-NEXT:    mov v6.s[2], wzr
; CHECK-GI-NEXT:    mov v7.s[3], w17
; CHECK-GI-NEXT:    mov v16.s[3], w18
; CHECK-GI-NEXT:    mov v17.s[3], w0
; CHECK-GI-NEXT:    mov v18.s[3], w1
; CHECK-GI-NEXT:    mov v19.s[3], w8
; CHECK-GI-NEXT:    mov v20.s[3], w9
; CHECK-GI-NEXT:    mov v21.s[3], w11
; CHECK-GI-NEXT:    mov v22.s[3], w13
; CHECK-GI-NEXT:    mov v23.s[3], wzr
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], wzr
; CHECK-GI-NEXT:    mov v6.s[3], wzr
; CHECK-GI-NEXT:    add v7.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v16.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v17.4s, v19.4s, v20.4s
; CHECK-GI-NEXT:    add v18.4s, v21.4s, v22.4s
; CHECK-GI-NEXT:    add v1.4s, v23.4s, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s
; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s
; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ldp x20, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <33 x i8>, ptr %a1
  %1 = zext <33 x i8> %0 to <33 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v33i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr b0, [x0, #32]
; CHECK-SD-NEXT:    ldr b1, [x1, #32]
; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q3, q4, [x1]
; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    ldp q1, q2, [x0]
; CHECK-SD-NEXT:    smull v5.8h, v4.8b, v2.8b
; CHECK-SD-NEXT:    smull v6.8h, v3.8b, v1.8b
; CHECK-SD-NEXT:    smull2 v2.8h, v4.16b, v2.16b
; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    smull2 v1.8h, v3.16b, v1.16b
; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
; CHECK-SD-NEXT:    saddl2 v3.4s, v6.8h, v5.8h
; CHECK-SD-NEXT:    saddl2 v0.4s, v1.8h, v2.8h
; CHECK-SD-NEXT:    saddl v1.4s, v1.4h, v2.4h
; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-SD-NEXT:    saddw v2.4s, v7.4s, v6.4h
; CHECK-SD-NEXT:    saddw v2.4s, v2.4s, v5.4h
; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v33i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sub sp, sp, #112
; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 112
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp q7, q16, [x1]
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill
; CHECK-GI-NEXT:    movi d3, #0000000000000000
; CHECK-GI-NEXT:    movi d2, #0000000000000000
; CHECK-GI-NEXT:    movi d5, #0000000000000000
; CHECK-GI-NEXT:    movi d4, #0000000000000000
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    mov b19, v7.b[3]
; CHECK-GI-NEXT:    mov b23, v7.b[7]
; CHECK-GI-NEXT:    mov b17, v7.b[1]
; CHECK-GI-NEXT:    fmov w11, s7
; CHECK-GI-NEXT:    mov b18, v7.b[2]
; CHECK-GI-NEXT:    mov b20, v7.b[4]
; CHECK-GI-NEXT:    mov b21, v7.b[5]
; CHECK-GI-NEXT:    mov b22, v7.b[6]
; CHECK-GI-NEXT:    mov b24, v7.b[8]
; CHECK-GI-NEXT:    mov b25, v7.b[9]
; CHECK-GI-NEXT:    mov b26, v7.b[10]
; CHECK-GI-NEXT:    mov b27, v7.b[11]
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov b28, v7.b[12]
; CHECK-GI-NEXT:    fmov w14, s19
; CHECK-GI-NEXT:    mov b19, v7.b[13]
; CHECK-GI-NEXT:    mov b29, v7.b[14]
; CHECK-GI-NEXT:    mov b7, v7.b[15]
; CHECK-GI-NEXT:    fmov w7, s23
; CHECK-GI-NEXT:    mov b23, v16.b[6]
; CHECK-GI-NEXT:    fmov w10, s17
; CHECK-GI-NEXT:    fmov w9, s18
; CHECK-GI-NEXT:    fmov w8, s24
; CHECK-GI-NEXT:    mov b30, v16.b[1]
; CHECK-GI-NEXT:    fmov w16, s25
; CHECK-GI-NEXT:    fmov w12, s20
; CHECK-GI-NEXT:    fmov w24, s21
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w7, w7
; CHECK-GI-NEXT:    fmov w22, s22
; CHECK-GI-NEXT:    stp s23, s7, [sp, #4] // 8-byte Folded Spill
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    fmov s7, w11
; CHECK-GI-NEXT:    mov b20, v16.b[2]
; CHECK-GI-NEXT:    mov b17, v16.b[3]
; CHECK-GI-NEXT:    mov b21, v16.b[4]
; CHECK-GI-NEXT:    mov b18, v16.b[5]
; CHECK-GI-NEXT:    fmov w27, s26
; CHECK-GI-NEXT:    fmov w25, s27
; CHECK-GI-NEXT:    mov b22, v16.b[7]
; CHECK-GI-NEXT:    fmov w26, s28
; CHECK-GI-NEXT:    mov v7.h[1], w10
; CHECK-GI-NEXT:    sxtb w10, w16
; CHECK-GI-NEXT:    mov b25, v16.b[8]
; CHECK-GI-NEXT:    fmov w23, s19
; CHECK-GI-NEXT:    mov b24, v16.b[9]
; CHECK-GI-NEXT:    fmov w5, s29
; CHECK-GI-NEXT:    mov b26, v16.b[10]
; CHECK-GI-NEXT:    mov b19, v16.b[11]
; CHECK-GI-NEXT:    fmov w6, s30
; CHECK-GI-NEXT:    mov b27, v16.b[12]
; CHECK-GI-NEXT:    mov b28, v16.b[13]
; CHECK-GI-NEXT:    mov b29, v16.b[14]
; CHECK-GI-NEXT:    sxtb w30, w23
; CHECK-GI-NEXT:    sxtb w5, w5
; CHECK-GI-NEXT:    mov v7.h[2], w9
; CHECK-GI-NEXT:    sxtb w9, w14
; CHECK-GI-NEXT:    fmov w20, s16
; CHECK-GI-NEXT:    mov b30, v16.b[15]
; CHECK-GI-NEXT:    fmov s16, w8
; CHECK-GI-NEXT:    sxtb w8, w12
; CHECK-GI-NEXT:    fmov w15, s17
; CHECK-GI-NEXT:    fmov w11, s18
; CHECK-GI-NEXT:    ldp q18, q17, [x0]
; CHECK-GI-NEXT:    mov v7.h[3], w9
; CHECK-GI-NEXT:    sxtb w9, w27
; CHECK-GI-NEXT:    fmov w18, s20
; CHECK-GI-NEXT:    sxtb w15, w15
; CHECK-GI-NEXT:    mov v16.h[1], w10
; CHECK-GI-NEXT:    sxtb w10, w25
; CHECK-GI-NEXT:    mov b20, v18.b[3]
; CHECK-GI-NEXT:    fmov w2, s22
; CHECK-GI-NEXT:    mov b22, v18.b[1]
; CHECK-GI-NEXT:    sxtb w18, w18
; CHECK-GI-NEXT:    fmov w13, s21
; CHECK-GI-NEXT:    mov b21, v18.b[2]
; CHECK-GI-NEXT:    mov v7.h[4], w8
; CHECK-GI-NEXT:    fmov w3, s19
; CHECK-GI-NEXT:    mov b19, v18.b[6]
; CHECK-GI-NEXT:    mov v16.h[2], w9
; CHECK-GI-NEXT:    sxtb w9, w24
; CHECK-GI-NEXT:    fmov w21, s25
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    fmov w28, s20
; CHECK-GI-NEXT:    mov b20, v18.b[11]
; CHECK-GI-NEXT:    fmov w8, s22
; CHECK-GI-NEXT:    mov b25, v18.b[8]
; CHECK-GI-NEXT:    fmov w29, s21
; CHECK-GI-NEXT:    mov v7.h[5], w9
; CHECK-GI-NEXT:    sxtb w9, w22
; CHECK-GI-NEXT:    fmov w19, s24
; CHECK-GI-NEXT:    mov v16.h[3], w10
; CHECK-GI-NEXT:    sxtb w10, w26
; CHECK-GI-NEXT:    fmov w26, s18
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w29, w29
; CHECK-GI-NEXT:    mov b24, v18.b[4]
; CHECK-GI-NEXT:    mov b23, v18.b[5]
; CHECK-GI-NEXT:    fmov w17, s27
; CHECK-GI-NEXT:    mov b27, v18.b[9]
; CHECK-GI-NEXT:    sxtb w23, w26
; CHECK-GI-NEXT:    mov v7.h[6], w9
; CHECK-GI-NEXT:    fmov w24, s19
; CHECK-GI-NEXT:    mov v16.h[4], w10
; CHECK-GI-NEXT:    mov b19, v18.b[14]
; CHECK-GI-NEXT:    fmov w10, s25
; CHECK-GI-NEXT:    fmov w4, s26
; CHECK-GI-NEXT:    fmov w16, s28
; CHECK-GI-NEXT:    mov b26, v18.b[7]
; CHECK-GI-NEXT:    mov b28, v18.b[10]
; CHECK-GI-NEXT:    fmov w27, s24
; CHECK-GI-NEXT:    mov b24, v18.b[12]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v7.h[7], w7
; CHECK-GI-NEXT:    fmov w7, s20
; CHECK-GI-NEXT:    sxtb w4, w4
; CHECK-GI-NEXT:    fmov s20, w23
; CHECK-GI-NEXT:    fmov w25, s23
; CHECK-GI-NEXT:    mov b23, v18.b[13]
; CHECK-GI-NEXT:    mov b22, v18.b[15]
; CHECK-GI-NEXT:    mov v16.h[5], w30
; CHECK-GI-NEXT:    sxtb w7, w7
; CHECK-GI-NEXT:    fmov w9, s27
; CHECK-GI-NEXT:    mov b21, v17.b[1]
; CHECK-GI-NEXT:    mov v20.h[1], w8
; CHECK-GI-NEXT:    sxtb w8, w20
; CHECK-GI-NEXT:    sxtb w20, w6
; CHECK-GI-NEXT:    fmov w6, s19
; CHECK-GI-NEXT:    fmov w26, s28
; CHECK-GI-NEXT:    mov b28, v17.b[8]
; CHECK-GI-NEXT:    fmov s18, w8
; CHECK-GI-NEXT:    sxtb w8, w21
; CHECK-GI-NEXT:    mov v16.h[6], w5
; CHECK-GI-NEXT:    fmov w5, s22
; CHECK-GI-NEXT:    fmov s22, w10
; CHECK-GI-NEXT:    sxtb w10, w27
; CHECK-GI-NEXT:    sxtb w26, w26
; CHECK-GI-NEXT:    mov v20.h[2], w29
; CHECK-GI-NEXT:    fmov s19, w8
; CHECK-GI-NEXT:    sxtb w8, w28
; CHECK-GI-NEXT:    sxtb w28, w19
; CHECK-GI-NEXT:    sxtb w19, w9
; CHECK-GI-NEXT:    fmov w27, s17
; CHECK-GI-NEXT:    mov b25, v17.b[2]
; CHECK-GI-NEXT:    fmov w29, s21
; CHECK-GI-NEXT:    mov b21, v17.b[9]
; CHECK-GI-NEXT:    mov v22.h[1], w19
; CHECK-GI-NEXT:    fmov w23, s23
; CHECK-GI-NEXT:    mov v20.h[3], w8
; CHECK-GI-NEXT:    mov b23, v17.b[6]
; CHECK-GI-NEXT:    fmov w30, s24
; CHECK-GI-NEXT:    sxtb w27, w27
; CHECK-GI-NEXT:    mov b24, v17.b[5]
; CHECK-GI-NEXT:    mov v18.h[1], w20
; CHECK-GI-NEXT:    fmov w21, s25
; CHECK-GI-NEXT:    mov b25, v17.b[10]
; CHECK-GI-NEXT:    mov v19.h[1], w28
; CHECK-GI-NEXT:    sxtb w28, w29
; CHECK-GI-NEXT:    mov v22.h[2], w26
; CHECK-GI-NEXT:    fmov w26, s21
; CHECK-GI-NEXT:    mov v20.h[4], w10
; CHECK-GI-NEXT:    fmov w10, s28
; CHECK-GI-NEXT:    fmov s21, w27
; CHECK-GI-NEXT:    sxtb w21, w21
; CHECK-GI-NEXT:    mov b27, v17.b[3]
; CHECK-GI-NEXT:    fmov w19, s23
; CHECK-GI-NEXT:    sxtb w26, w26
; CHECK-GI-NEXT:    fmov w22, s26
; CHECK-GI-NEXT:    mov b26, v17.b[4]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v21.h[1], w28
; CHECK-GI-NEXT:    fmov w8, s24
; CHECK-GI-NEXT:    mov b24, v17.b[11]
; CHECK-GI-NEXT:    fmov w27, s25
; CHECK-GI-NEXT:    mov v18.h[2], w18
; CHECK-GI-NEXT:    sxtb w18, w25
; CHECK-GI-NEXT:    fmov s23, w10
; CHECK-GI-NEXT:    fmov w20, s27
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    fmov w9, s26
; CHECK-GI-NEXT:    mov b26, v17.b[12]
; CHECK-GI-NEXT:    sxtb w25, w27
; CHECK-GI-NEXT:    mov v20.h[5], w18
; CHECK-GI-NEXT:    sxtb w18, w3
; CHECK-GI-NEXT:    sxtb w3, w24
; CHECK-GI-NEXT:    mov v23.h[1], w26
; CHECK-GI-NEXT:    mov v21.h[2], w21
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    fmov w28, s24
; CHECK-GI-NEXT:    mov v22.h[3], w7
; CHECK-GI-NEXT:    sxtb w7, w20
; CHECK-GI-NEXT:    mov v19.h[2], w4
; CHECK-GI-NEXT:    sxtb w4, w30
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v18.h[3], w15
; CHECK-GI-NEXT:    sxtb w20, w28
; CHECK-GI-NEXT:    sxtb w15, w17
; CHECK-GI-NEXT:    sxtb w17, w22
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v23.h[2], w25
; CHECK-GI-NEXT:    mov v20.h[6], w3
; CHECK-GI-NEXT:    mov v21.h[3], w7
; CHECK-GI-NEXT:    fmov w10, s26
; CHECK-GI-NEXT:    mov v22.h[4], w4
; CHECK-GI-NEXT:    mov v19.h[3], w18
; CHECK-GI-NEXT:    sxtb w18, w23
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov b27, v17.b[13]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v23.h[3], w20
; CHECK-GI-NEXT:    mov v18.h[4], w13
; CHECK-GI-NEXT:    sxtb w13, w6
; CHECK-GI-NEXT:    mov v20.h[7], w17
; CHECK-GI-NEXT:    mov v21.h[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #8] // 4-byte Reload
; CHECK-GI-NEXT:    mov v22.h[5], w18
; CHECK-GI-NEXT:    mov b25, v17.b[14]
; CHECK-GI-NEXT:    fmov w26, s27
; CHECK-GI-NEXT:    mov v19.h[4], w15
; CHECK-GI-NEXT:    fmov w14, s29
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v23.h[4], w10
; CHECK-GI-NEXT:    sxtb w10, w11
; CHECK-GI-NEXT:    sxtb w11, w16
; CHECK-GI-NEXT:    mov v21.h[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #4] // 4-byte Reload
; CHECK-GI-NEXT:    sxtb w15, w26
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v18.h[5], w10
; CHECK-GI-NEXT:    sxtb w10, w19
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mul v20.8h, v7.8h, v20.8h
; CHECK-GI-NEXT:    mov b7, v17.b[7]
; CHECK-GI-NEXT:    mov v22.h[6], w13
; CHECK-GI-NEXT:    sxtb w13, w5
; CHECK-GI-NEXT:    fmov w27, s25
; CHECK-GI-NEXT:    mov v19.h[5], w11
; CHECK-GI-NEXT:    sxtb w11, w2
; CHECK-GI-NEXT:    mov b17, v17.b[15]
; CHECK-GI-NEXT:    mov v18.h[6], w8
; CHECK-GI-NEXT:    mov v16.h[7], w9
; CHECK-GI-NEXT:    sxtb w9, w14
; CHECK-GI-NEXT:    mov v23.h[5], w15
; CHECK-GI-NEXT:    mov v21.h[6], w10
; CHECK-GI-NEXT:    sxtb w14, w27
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    fmov w8, s7
; CHECK-GI-NEXT:    mov v22.h[7], w13
; CHECK-GI-NEXT:    fmov w12, s30
; CHECK-GI-NEXT:    mov v19.h[6], w9
; CHECK-GI-NEXT:    fmov w9, s17
; CHECK-GI-NEXT:    smov w10, v20.h[0]
; CHECK-GI-NEXT:    mov v23.h[6], w14
; CHECK-GI-NEXT:    mov v18.h[7], w11
; CHECK-GI-NEXT:    smov w13, v20.h[1]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    smov w11, v20.h[4]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mul v22.8h, v16.8h, v22.8h
; CHECK-GI-NEXT:    smov w14, v20.h[3]
; CHECK-GI-NEXT:    mov v21.h[7], w8
; CHECK-GI-NEXT:    ldrsb w8, [x0, #32]
; CHECK-GI-NEXT:    mov v19.h[7], w12
; CHECK-GI-NEXT:    mov v23.h[7], w9
; CHECK-GI-NEXT:    ldrsb w9, [x1, #32]
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    smov w10, v20.h[2]
; CHECK-GI-NEXT:    smov w12, v20.h[5]
; CHECK-GI-NEXT:    fmov s16, w11
; CHECK-GI-NEXT:    mul w9, w9, w8
; CHECK-GI-NEXT:    smov w15, v22.h[4]
; CHECK-GI-NEXT:    smov w17, v22.h[5]
; CHECK-GI-NEXT:    mul v24.8h, v18.8h, v21.8h
; CHECK-GI-NEXT:    mov v7.s[1], w13
; CHECK-GI-NEXT:    smov w13, v22.h[0]
; CHECK-GI-NEXT:    mul v18.8h, v19.8h, v23.8h
; CHECK-GI-NEXT:    smov w16, v22.h[1]
; CHECK-GI-NEXT:    smov w8, v20.h[7]
; CHECK-GI-NEXT:    sxth w9, w9
; CHECK-GI-NEXT:    mov v16.s[1], w12
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    fmov s19, w15
; CHECK-GI-NEXT:    smov w15, v22.h[6]
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    smov w11, v24.h[0]
; CHECK-GI-NEXT:    mov v7.s[2], w10
; CHECK-GI-NEXT:    smov w10, v20.h[6]
; CHECK-GI-NEXT:    smov w12, v24.h[1]
; CHECK-GI-NEXT:    smov w0, v18.h[4]
; CHECK-GI-NEXT:    fmov s17, w13
; CHECK-GI-NEXT:    mov v19.s[1], w17
; CHECK-GI-NEXT:    smov w17, v18.h[0]
; CHECK-GI-NEXT:    smov w18, v18.h[1]
; CHECK-GI-NEXT:    smov w13, v22.h[2]
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    fmov s20, w11
; CHECK-GI-NEXT:    smov w11, v24.h[4]
; CHECK-GI-NEXT:    mov v7.s[3], w14
; CHECK-GI-NEXT:    smov w14, v24.h[5]
; CHECK-GI-NEXT:    mov v17.s[1], w16
; CHECK-GI-NEXT:    smov w16, v24.h[2]
; CHECK-GI-NEXT:    mov v19.s[2], w15
; CHECK-GI-NEXT:    smov w15, v18.h[5]
; CHECK-GI-NEXT:    fmov s23, w0
; CHECK-GI-NEXT:    mov v20.s[1], w12
; CHECK-GI-NEXT:    mov v16.s[2], w10
; CHECK-GI-NEXT:    smov w10, v22.h[3]
; CHECK-GI-NEXT:    fmov s21, w11
; CHECK-GI-NEXT:    smov w11, v22.h[7]
; CHECK-GI-NEXT:    fmov s22, w17
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    mov v4.s[1], wzr
; CHECK-GI-NEXT:    mov v6.s[1], wzr
; CHECK-GI-NEXT:    mov v23.s[1], w15
; CHECK-GI-NEXT:    smov w15, v18.h[6]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v21.s[1], w14
; CHECK-GI-NEXT:    smov w14, v24.h[6]
; CHECK-GI-NEXT:    mov v20.s[2], w16
; CHECK-GI-NEXT:    mov v22.s[1], w18
; CHECK-GI-NEXT:    smov w16, v18.h[2]
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], wzr
; CHECK-GI-NEXT:    mov v6.s[2], wzr
; CHECK-GI-NEXT:    mov v23.s[2], w15
; CHECK-GI-NEXT:    mov v21.s[2], w14
; CHECK-GI-NEXT:    smov w14, v18.h[3]
; CHECK-GI-NEXT:    smov w15, v18.h[7]
; CHECK-GI-NEXT:    fmov s18, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload
; CHECK-GI-NEXT:    mov v17.s[2], w13
; CHECK-GI-NEXT:    smov w12, v24.h[3]
; CHECK-GI-NEXT:    smov w13, v24.h[7]
; CHECK-GI-NEXT:    mov v22.s[2], w16
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], wzr
; CHECK-GI-NEXT:    mov v6.s[3], wzr
; CHECK-GI-NEXT:    mov v18.s[1], wzr
; CHECK-GI-NEXT:    mov v16.s[3], w8
; CHECK-GI-NEXT:    mov v17.s[3], w10
; CHECK-GI-NEXT:    mov v19.s[3], w11
; CHECK-GI-NEXT:    mov v20.s[3], w12
; CHECK-GI-NEXT:    mov v21.s[3], w13
; CHECK-GI-NEXT:    mov v22.s[3], w14
; CHECK-GI-NEXT:    mov v23.s[3], w15
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s
; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s
; CHECK-GI-NEXT:    mov v18.s[2], wzr
; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v5.4s, v17.4s, v19.4s
; CHECK-GI-NEXT:    add v6.4s, v20.4s, v21.4s
; CHECK-GI-NEXT:    add v7.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    mov v18.s[3], wzr
; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v18.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    add sp, sp, #112
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <33 x i8>, ptr %a
  %1 = sext <33 x i8> %0 to <33 x i32>
  %2 = load <33 x i8>, ptr %b
  %3 = sext <33 x i8> %2 to <33 x i32>
  %4 = mul nsw <33 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v33i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    ldr b0, [sp, #344]
; CHECK-SD-NEXT:    add x8, sp, #352
; CHECK-SD-NEXT:    ldr b1, [sp, #80]
; CHECK-SD-NEXT:    ldr b2, [sp, #216]
; CHECK-SD-NEXT:    add x9, sp, #96
; CHECK-SD-NEXT:    add x10, sp, #104
; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #88
; CHECK-SD-NEXT:    ldr b4, [sp, #408]
; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #360
; CHECK-SD-NEXT:    add x12, sp, #248
; CHECK-SD-NEXT:    add x13, sp, #432
; CHECK-SD-NEXT:    add x11, sp, #384
; CHECK-SD-NEXT:    ldr b5, [sp, #144]
; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #224
; CHECK-SD-NEXT:    ldr b6, [sp, #280]
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
; CHECK-SD-NEXT:    add x8, sp, #368
; CHECK-SD-NEXT:    add x9, sp, #232
; CHECK-SD-NEXT:    ldr b16, [sp, #744]
; CHECK-SD-NEXT:    ldr b17, [sp, #480]
; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #376
; CHECK-SD-NEXT:    ldr b18, [sp, #936]
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x10]
; CHECK-SD-NEXT:    add x9, sp, #240
; CHECK-SD-NEXT:    add x10, sp, #392
; CHECK-SD-NEXT:    ldr b19, [sp, #672]
; CHECK-SD-NEXT:    ldr b7, [sp, #16]
; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #112
; CHECK-SD-NEXT:    ldr b21, [sp, #1000]
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #416
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #120
; CHECK-SD-NEXT:    add x9, sp, #400
; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #128
; CHECK-SD-NEXT:    ldr b22, [sp, #736]
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
; CHECK-SD-NEXT:    add x12, sp, #424
; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
; CHECK-SD-NEXT:    add x12, sp, #152
; CHECK-SD-NEXT:    add x8, sp, #136
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
; CHECK-SD-NEXT:    add x12, sp, #440
; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
; CHECK-SD-NEXT:    add x11, sp, #288
; CHECK-SD-NEXT:    add x10, sp, #256
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #296
; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #160
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]
; CHECK-SD-NEXT:    add x10, sp, #168
; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x12]
; CHECK-SD-NEXT:    add x12, sp, #448
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x11]
; CHECK-SD-NEXT:    add x11, sp, #304
; CHECK-SD-NEXT:    add x8, sp, #464
; CHECK-SD-NEXT:    add x13, sp, #768
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x10]
; CHECK-SD-NEXT:    add x10, sp, #176
; CHECK-SD-NEXT:    add x9, sp, #264
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x12]
; CHECK-SD-NEXT:    add x12, sp, #456
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #760
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #272
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #312
; CHECK-SD-NEXT:    fmov s3, w0
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #320
; CHECK-SD-NEXT:    add x12, sp, #680
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #184
; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x12]
; CHECK-SD-NEXT:    add x12, sp, #776
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
; CHECK-SD-NEXT:    add x8, sp, #752
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x8]
; CHECK-SD-NEXT:    add x10, sp, #24
; CHECK-SD-NEXT:    smull v22.8h, v22.8b, v21.8b
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
; CHECK-SD-NEXT:    add x10, sp, #496
; CHECK-SD-NEXT:    mov v3.b[1], w1
; CHECK-SD-NEXT:    add x9, sp, #192
; CHECK-SD-NEXT:    ldr b20, [sp, #472]
; CHECK-SD-NEXT:    ldr b23, [sp, #208]
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
; CHECK-SD-NEXT:    add x11, sp, #488
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #944
; CHECK-SD-NEXT:    add x9, sp, #328
; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #688
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]
; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x11]
; CHECK-SD-NEXT:    add x11, sp, #504
; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #952
; CHECK-SD-NEXT:    add x13, sp, #784
; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #32
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x12]
; CHECK-SD-NEXT:    add x12, sp, #696
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #960
; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x12]
; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x11]
; CHECK-SD-NEXT:    add x10, sp, #512
; CHECK-SD-NEXT:    add x11, sp, #704
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x13]
; CHECK-SD-NEXT:    add x12, sp, #792
; CHECK-SD-NEXT:    sshll v24.4s, v22.4h, #0
; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #968
; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x11]
; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #520
; CHECK-SD-NEXT:    add x11, sp, #976
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
; CHECK-SD-NEXT:    add x12, sp, #712
; CHECK-SD-NEXT:    smull v20.8h, v23.8b, v20.8b
; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x12]
; CHECK-SD-NEXT:    add x12, sp, #720
; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #528
; CHECK-SD-NEXT:    add x10, sp, #800
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
; CHECK-SD-NEXT:    add x10, sp, #536
; CHECK-SD-NEXT:    ldr b22, [sp, #872]
; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x11]
; CHECK-SD-NEXT:    add x11, sp, #984
; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
; CHECK-SD-NEXT:    add x11, sp, #992
; CHECK-SD-NEXT:    add x12, sp, #728
; CHECK-SD-NEXT:    ldr b23, [sp, #608]
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #880
; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x12]
; CHECK-SD-NEXT:    add x10, sp, #816
; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x11]
; CHECK-SD-NEXT:    add x11, sp, #552
; CHECK-SD-NEXT:    add x12, sp, #616
; CHECK-SD-NEXT:    mov v3.b[2], w2
; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x9]
; CHECK-SD-NEXT:    ld1 { v23.b }[1], [x12]
; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
; CHECK-SD-NEXT:    add x12, sp, #560
; CHECK-SD-NEXT:    add x9, sp, #888
; CHECK-SD-NEXT:    smull v17.8h, v19.8b, v18.8b
; CHECK-SD-NEXT:    ldr b18, [sp, #808]
; CHECK-SD-NEXT:    ldr b19, [sp, #544]
; CHECK-SD-NEXT:    add x13, sp, #624
; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #896
; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #824
; CHECK-SD-NEXT:    add x10, sp, #48
; CHECK-SD-NEXT:    ld1 { v23.b }[2], [x13]
; CHECK-SD-NEXT:    mov v3.b[3], w3
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #832
; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
; CHECK-SD-NEXT:    add x11, sp, #568
; CHECK-SD-NEXT:    add x12, sp, #632
; CHECK-SD-NEXT:    add x9, sp, #904
; CHECK-SD-NEXT:    add x13, sp, #640
; CHECK-SD-NEXT:    ld1 { v23.b }[3], [x12]
; CHECK-SD-NEXT:    add x12, sp, #576
; CHECK-SD-NEXT:    mov v3.b[4], w4
; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #840
; CHECK-SD-NEXT:    add x10, sp, #56
; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x9]
; CHECK-SD-NEXT:    add x9, sp, #912
; CHECK-SD-NEXT:    ld1 { v23.b }[4], [x13]
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
; CHECK-SD-NEXT:    add x10, sp, #848
; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
; CHECK-SD-NEXT:    add x11, sp, #584
; CHECK-SD-NEXT:    add x12, sp, #648
; CHECK-SD-NEXT:    mov v3.b[5], w5
; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x9]
; CHECK-SD-NEXT:    ld1 { v23.b }[5], [x12]
; CHECK-SD-NEXT:    add x12, sp, #592
; CHECK-SD-NEXT:    movi v21.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #856
; CHECK-SD-NEXT:    add x9, sp, #920
; CHECK-SD-NEXT:    add x13, sp, #656
; CHECK-SD-NEXT:    add x10, sp, #64
; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v23.b }[6], [x13]
; CHECK-SD-NEXT:    mov v3.b[6], w6
; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
; CHECK-SD-NEXT:    add x10, sp, #864
; CHECK-SD-NEXT:    add x11, sp, #600
; CHECK-SD-NEXT:    add x9, sp, #928
; CHECK-SD-NEXT:    add x12, sp, #664
; CHECK-SD-NEXT:    mov v21.s[0], v24.s[0]
; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v23.b }[7], [x12]
; CHECK-SD-NEXT:    add x8, sp, #200
; CHECK-SD-NEXT:    mov v3.b[7], w7
; CHECK-SD-NEXT:    add x10, sp, #336
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
; CHECK-SD-NEXT:    add x8, sp, #72
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x10]
; CHECK-SD-NEXT:    smull v18.8h, v19.8b, v18.8b
; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v22.8b
; CHECK-SD-NEXT:    sshll v20.4s, v20.4h, #0
; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT:    saddw v1.4s, v21.4s, v16.4h
; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v2.8b
; CHECK-SD-NEXT:    smull v3.8h, v5.8b, v4.8b
; CHECK-SD-NEXT:    smull v4.8h, v7.8b, v6.8b
; CHECK-SD-NEXT:    mov v19.s[0], v20.s[0]
; CHECK-SD-NEXT:    saddl2 v5.4s, v18.8h, v17.8h
; CHECK-SD-NEXT:    saddl v7.4s, v18.4h, v17.4h
; CHECK-SD-NEXT:    saddl2 v6.4s, v16.8h, v22.8h
; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v22.4h
; CHECK-SD-NEXT:    saddl2 v17.4s, v2.8h, v0.8h
; CHECK-SD-NEXT:    saddl2 v16.4s, v4.8h, v3.8h
; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
; CHECK-SD-NEXT:    saddw v2.4s, v19.4s, v2.4h
; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v7.4s
; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v33i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    str x29, [sp, #64] // 8-byte Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    .cfi_offset b8, -24
; CHECK-GI-NEXT:    .cfi_offset b9, -32
; CHECK-GI-NEXT:    .cfi_offset b10, -40
; CHECK-GI-NEXT:    .cfi_offset b11, -48
; CHECK-GI-NEXT:    .cfi_offset b12, -56
; CHECK-GI-NEXT:    .cfi_offset b13, -64
; CHECK-GI-NEXT:    .cfi_offset b14, -72
; CHECK-GI-NEXT:    .cfi_offset b15, -80
; CHECK-GI-NEXT:    lsl w8, w0, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #80]
; CHECK-GI-NEXT:    lsl w11, w1, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #88]
; CHECK-GI-NEXT:    ldr w13, [sp, #128]
; CHECK-GI-NEXT:    ldr w14, [sp, #136]
; CHECK-GI-NEXT:    sbfx w12, w8, #8, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w8, w11, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    lsl w11, w2, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    fmov s22, w12
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    ldr w12, [sp, #152]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    lsl w16, w7, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    fmov s23, w10
; CHECK-GI-NEXT:    sbfx w10, w11, #8, #8
; CHECK-GI-NEXT:    lsl w11, w3, #8
; CHECK-GI-NEXT:    mov v22.h[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #96]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #176]
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    mov v23.h[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #104]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    ldr w17, [sp, #224]
; CHECK-GI-NEXT:    mov v22.h[2], w10
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w11, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    lsl w11, w4, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    mov v23.h[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #112]
; CHECK-GI-NEXT:    movi d19, #0000000000000000
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    movi d21, #0000000000000000
; CHECK-GI-NEXT:    mov v22.h[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #144]
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    movi d16, #0000000000000000
; CHECK-GI-NEXT:    movi d18, #0000000000000000
; CHECK-GI-NEXT:    movi d17, #0000000000000000
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v23.h[3], w9
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #120]
; CHECK-GI-NEXT:    movi d20, #0000000000000000
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    mov v22.h[4], w11
; CHECK-GI-NEXT:    lsl w11, w5, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    movi d7, #0000000000000000
; CHECK-GI-NEXT:    movi d2, #0000000000000000
; CHECK-GI-NEXT:    fmov s24, w10
; CHECK-GI-NEXT:    mov v23.h[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #160]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #168]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    movi d4, #0000000000000000
; CHECK-GI-NEXT:    movi d3, #0000000000000000
; CHECK-GI-NEXT:    mov v24.h[1], w12
; CHECK-GI-NEXT:    lsl w12, w6, #8
; CHECK-GI-NEXT:    mov v22.h[5], w11
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v23.h[5], w9
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #184]
; CHECK-GI-NEXT:    ldr w9, [sp, #192]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    movi d5, #0000000000000000
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    mov v24.h[2], w8
; CHECK-GI-NEXT:    mov v22.h[6], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #208]
; CHECK-GI-NEXT:    mov v23.h[6], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #216]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    ldr w8, [sp, #200]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v19.s[1], wzr
; CHECK-GI-NEXT:    mov v24.h[3], w10
; CHECK-GI-NEXT:    sbfx w10, w14, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #280]
; CHECK-GI-NEXT:    mov v22.h[7], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #288]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v23.h[7], w10
; CHECK-GI-NEXT:    lsl w18, w16, #8
; CHECK-GI-NEXT:    fmov s27, w12
; CHECK-GI-NEXT:    ldr w10, [sp, #232]
; CHECK-GI-NEXT:    sbfx w16, w14, #8, #8
; CHECK-GI-NEXT:    mov v24.h[4], w15
; CHECK-GI-NEXT:    lsl w15, w11, #8
; CHECK-GI-NEXT:    sbfx w14, w18, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #296]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    fmov s25, w16
; CHECK-GI-NEXT:    ldr w16, [sp, #344]
; CHECK-GI-NEXT:    mov v27.h[1], w13
; CHECK-GI-NEXT:    lsl w13, w17, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    ldr w12, [sp, #240]
; CHECK-GI-NEXT:    sbfx w17, w10, #8, #8
; CHECK-GI-NEXT:    mov v25.h[1], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #352]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v24.h[5], w15
; CHECK-GI-NEXT:    mov v27.h[2], w13
; CHECK-GI-NEXT:    lsl w13, w14, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #304]
; CHECK-GI-NEXT:    fmov s26, w16
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #248]
; CHECK-GI-NEXT:    mov v25.h[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #360]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    mov v24.h[6], w9
; CHECK-GI-NEXT:    lsl w16, w11, #8
; CHECK-GI-NEXT:    mov v26.h[1], w13
; CHECK-GI-NEXT:    mov v27.h[3], w17
; CHECK-GI-NEXT:    sbfx w13, w14, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #312]
; CHECK-GI-NEXT:    ldr w17, [sp, #328]
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #256]
; CHECK-GI-NEXT:    ldr w11, [sp, #264]
; CHECK-GI-NEXT:    mov v25.h[3], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #368]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    mov v26.h[2], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #320]
; CHECK-GI-NEXT:    mov v27.h[4], w12
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w9, w14, #8, #8
; CHECK-GI-NEXT:    lsl w14, w15, #8
; CHECK-GI-NEXT:    lsl w15, w16, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #408]
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w12, w13, #8, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #376]
; CHECK-GI-NEXT:    mov v25.h[4], w9
; CHECK-GI-NEXT:    sbfx w9, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w14, w15, #8, #8
; CHECK-GI-NEXT:    lsl w15, w16, #8
; CHECK-GI-NEXT:    mov v26.h[3], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #416]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w16, w17, #8
; CHECK-GI-NEXT:    mov v27.h[5], w9
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v25.h[5], w14
; CHECK-GI-NEXT:    fmov s29, w15
; CHECK-GI-NEXT:    ldr w14, [sp, #384]
; CHECK-GI-NEXT:    ldr w15, [sp, #472]
; CHECK-GI-NEXT:    mov v26.h[4], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #424]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v29.h[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #480]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v25.h[6], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #432]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v26.h[5], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #392]
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    mov v29.h[2], w13
; CHECK-GI-NEXT:    fmov s28, w15
; CHECK-GI-NEXT:    ldr w9, [sp, #336]
; CHECK-GI-NEXT:    ldr w13, [sp, #488]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    ldr w15, [sp, #440]
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v28.h[1], w12
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    mov v29.h[3], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #496]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    ldr w12, [sp, #400]
; CHECK-GI-NEXT:    mov v26.h[6], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #448]
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    mov v28.h[2], w13
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    mov v25.h[7], w9
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v29.h[4], w15
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #456]
; CHECK-GI-NEXT:    ldr w15, [sp, #504]
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w12, w14, #8, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    lsl w14, w15, #8
; CHECK-GI-NEXT:    mov v28.h[3], w16
; CHECK-GI-NEXT:    ldr w15, [sp, #512]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mul v30.8h, v22.8h, v25.8h
; CHECK-GI-NEXT:    mov v26.h[7], w9
; CHECK-GI-NEXT:    mov v29.h[5], w12
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    sbfx w9, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w14, w11, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w13, #8, #8
; CHECK-GI-NEXT:    lsl w13, w15, #8
; CHECK-GI-NEXT:    ldr w17, [sp, #464]
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v28.h[4], w9
; CHECK-GI-NEXT:    mov v27.h[6], w10
; CHECK-GI-NEXT:    ldr w16, [sp, #520]
; CHECK-GI-NEXT:    sbfx w10, w13, #8, #8
; CHECK-GI-NEXT:    smov w13, v30.h[0]
; CHECK-GI-NEXT:    mov v24.h[7], w8
; CHECK-GI-NEXT:    lsl w8, w17, #8
; CHECK-GI-NEXT:    mov v29.h[6], w11
; CHECK-GI-NEXT:    mul v26.8h, v23.8h, v26.8h
; CHECK-GI-NEXT:    lsl w15, w16, #8
; CHECK-GI-NEXT:    smov w16, v30.h[1]
; CHECK-GI-NEXT:    ldr w12, [sp, #528]
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v28.h[5], w10
; CHECK-GI-NEXT:    mov v27.h[7], w14
; CHECK-GI-NEXT:    fmov s22, w13
; CHECK-GI-NEXT:    sbfx w10, w15, #8, #8
; CHECK-GI-NEXT:    smov w14, v30.h[4]
; CHECK-GI-NEXT:    mov v29.h[7], w8
; CHECK-GI-NEXT:    smov w15, v26.h[0]
; CHECK-GI-NEXT:    smov w13, v30.h[2]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #544]
; CHECK-GI-NEXT:    ldr w11, [sp, #552]
; CHECK-GI-NEXT:    mov v22.s[1], w16
; CHECK-GI-NEXT:    smov w16, v26.h[4]
; CHECK-GI-NEXT:    mov v28.h[6], w10
; CHECK-GI-NEXT:    smov w10, v26.h[1]
; CHECK-GI-NEXT:    fmov s23, w14
; CHECK-GI-NEXT:    smov w14, v26.h[5]
; CHECK-GI-NEXT:    mul v29.8h, v24.8h, v29.8h
; CHECK-GI-NEXT:    fmov s24, w15
; CHECK-GI-NEXT:    smov w15, v26.h[2]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    smov w8, v30.h[5]
; CHECK-GI-NEXT:    smov w17, v30.h[7]
; CHECK-GI-NEXT:    fmov s25, w16
; CHECK-GI-NEXT:    mov v22.s[2], w13
; CHECK-GI-NEXT:    smov w13, v30.h[3]
; CHECK-GI-NEXT:    mov v24.s[1], w10
; CHECK-GI-NEXT:    smov w16, v26.h[6]
; CHECK-GI-NEXT:    sbfx w10, w12, #8, #8
; CHECK-GI-NEXT:    smov w18, v29.h[0]
; CHECK-GI-NEXT:    smov w0, v29.h[1]
; CHECK-GI-NEXT:    ldr w12, [sp, #560]
; CHECK-GI-NEXT:    mov v25.s[1], w14
; CHECK-GI-NEXT:    smov w14, v26.h[7]
; CHECK-GI-NEXT:    mov v28.h[7], w10
; CHECK-GI-NEXT:    mov v22.s[3], w13
; CHECK-GI-NEXT:    smov w13, v26.h[3]
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v24.s[2], w15
; CHECK-GI-NEXT:    smov w15, v29.h[2]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    fmov s26, w18
; CHECK-GI-NEXT:    mov v23.s[1], w8
; CHECK-GI-NEXT:    smov w8, v30.h[6]
; CHECK-GI-NEXT:    mov v25.s[2], w16
; CHECK-GI-NEXT:    lsl w16, w9, #8
; CHECK-GI-NEXT:    mul v31.8h, v27.8h, v28.8h
; CHECK-GI-NEXT:    ldr w10, [sp, #568]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #584]
; CHECK-GI-NEXT:    mov v24.s[3], w13
; CHECK-GI-NEXT:    smov w13, v29.h[4]
; CHECK-GI-NEXT:    mov v26.s[1], w0
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v23.s[2], w8
; CHECK-GI-NEXT:    mov v25.s[3], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #608]
; CHECK-GI-NEXT:    ldr w8, [sp, #576]
; CHECK-GI-NEXT:    fmov s8, w16
; CHECK-GI-NEXT:    ldr w16, [sp, #616]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    fmov s27, w13
; CHECK-GI-NEXT:    lsl w13, w14, #8
; CHECK-GI-NEXT:    mov v26.s[2], w15
; CHECK-GI-NEXT:    smov w15, v29.h[5]
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #624]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v8.h[1], w11
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    mov v23.s[3], w17
; CHECK-GI-NEXT:    fmov s9, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #632]
; CHECK-GI-NEXT:    smov w17, v31.h[1]
; CHECK-GI-NEXT:    mov v27.s[1], w15
; CHECK-GI-NEXT:    smov w15, v31.h[0]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v8.h[2], w12
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v9.h[1], w16
; CHECK-GI-NEXT:    smov w16, v31.h[2]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #592]
; CHECK-GI-NEXT:    ldr w12, [sp, #600]
; CHECK-GI-NEXT:    fmov s28, w15
; CHECK-GI-NEXT:    smov w15, v29.h[6]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v8.h[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #640]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v9.h[2], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #672]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v28.s[1], w17
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    mov v27.s[2], w15
; CHECK-GI-NEXT:    ldr w15, [sp, #680]
; CHECK-GI-NEXT:    mov v8.h[4], w8
; CHECK-GI-NEXT:    smov w8, v31.h[4]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    mov v9.h[3], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #688]
; CHECK-GI-NEXT:    mov v28.s[2], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #648]
; CHECK-GI-NEXT:    fmov s10, w14
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #656]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    fmov s30, w8
; CHECK-GI-NEXT:    sbfx w8, w10, #8, #8
; CHECK-GI-NEXT:    smov w10, v31.h[5]
; CHECK-GI-NEXT:    mov v8.h[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #696]
; CHECK-GI-NEXT:    mov v10.h[1], w15
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v9.h[4], w8
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    ldr w8, [sp, #704]
; CHECK-GI-NEXT:    ldr w15, [sp, #664]
; CHECK-GI-NEXT:    ldr w17, [sp, #768]
; CHECK-GI-NEXT:    mov v30.s[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #744]
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    mov v10.h[2], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #736]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v9.h[5], w16
; CHECK-GI-NEXT:    mov v8.h[6], w11
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #712]
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #720]
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v10.h[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #752]
; CHECK-GI-NEXT:    mov v8.h[7], w12
; CHECK-GI-NEXT:    sbfx w12, w8, #8, #8
; CHECK-GI-NEXT:    lsl w18, w16, #8
; CHECK-GI-NEXT:    fmov s11, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #760]
; CHECK-GI-NEXT:    ldr w8, [sp, #784]
; CHECK-GI-NEXT:    mov v21.s[1], wzr
; CHECK-GI-NEXT:    mov v16.s[1], wzr
; CHECK-GI-NEXT:    mov v18.s[1], wzr
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v10.h[4], w12
; CHECK-GI-NEXT:    sbfx w12, w15, #8, #8
; CHECK-GI-NEXT:    mov v11.h[1], w10
; CHECK-GI-NEXT:    sbfx w10, w14, #8, #8
; CHECK-GI-NEXT:    lsl w14, w9, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    ldr w9, [sp, #776]
; CHECK-GI-NEXT:    lsl w8, w8, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v9.h[6], w10
; CHECK-GI-NEXT:    lsl w10, w11, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #808]
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v11.h[2], w14
; CHECK-GI-NEXT:    ldr w14, [sp, #816]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    mov v17.s[1], wzr
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    mov v9.h[7], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #824]
; CHECK-GI-NEXT:    sbfx w16, w11, #8, #8
; CHECK-GI-NEXT:    mov v10.h[5], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #832]
; CHECK-GI-NEXT:    mov v11.h[3], w13
; CHECK-GI-NEXT:    sbfx w15, w14, #8, #8
; CHECK-GI-NEXT:    lsl w14, w17, #8
; CHECK-GI-NEXT:    fmov s12, w16
; CHECK-GI-NEXT:    ldr w16, [sp, #872]
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #840]
; CHECK-GI-NEXT:    sbfx w13, w18, #8, #8
; CHECK-GI-NEXT:    sbfx w17, w12, #8, #8
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    ldr w12, [sp, #856]
; CHECK-GI-NEXT:    mov v12.h[1], w15
; CHECK-GI-NEXT:    mov v11.h[4], w14
; CHECK-GI-NEXT:    ldr w15, [sp, #880]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v10.h[6], w13
; CHECK-GI-NEXT:    ldr w13, [sp, #848]
; CHECK-GI-NEXT:    lsl w14, w15, #8
; CHECK-GI-NEXT:    sbfx w15, w16, #8, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #888]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    mov v20.s[1], wzr
; CHECK-GI-NEXT:    mov v12.h[2], w17
; CHECK-GI-NEXT:    lsl w17, w10, #8
; CHECK-GI-NEXT:    mov v11.h[5], w9
; CHECK-GI-NEXT:    fmov s13, w15
; CHECK-GI-NEXT:    ldr w9, [sp, #936]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    sbfx w15, w17, #8, #8
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    lsl w9, w9, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    ldr w10, [sp, #864]
; CHECK-GI-NEXT:    mov v12.h[3], w15
; CHECK-GI-NEXT:    mov v11.h[6], w8
; CHECK-GI-NEXT:    sbfx w8, w11, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #1000]
; CHECK-GI-NEXT:    mov v13.h[1], w14
; CHECK-GI-NEXT:    ldr w15, [sp, #944]
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #896]
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    mov v12.h[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1008]
; CHECK-GI-NEXT:    fmov s14, w9
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v13.h[2], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #952]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    lsl w17, w8, #8
; CHECK-GI-NEXT:    smov w8, v29.h[3]
; CHECK-GI-NEXT:    smov w9, v29.h[7]
; CHECK-GI-NEXT:    fmov s29, w11
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v14.h[1], w15
; CHECK-GI-NEXT:    sbfx w15, w17, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #904]
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    mov v12.h[5], w13
; CHECK-GI-NEXT:    mov v13.h[3], w14
; CHECK-GI-NEXT:    mov v29.h[1], w15
; CHECK-GI-NEXT:    ldr w15, [sp, #960]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #1016]
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #1024]
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v14.h[2], w16
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    ldr w16, [sp, #912]
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v13.h[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #968]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v12.h[6], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #976]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mov v14.h[3], w15
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v29.h[2], w14
; CHECK-GI-NEXT:    ldr w15, [sp, #1032]
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    ldr w14, [sp, #920]
; CHECK-GI-NEXT:    mov v26.s[3], w8
; CHECK-GI-NEXT:    sbfx w16, w16, #8, #8
; CHECK-GI-NEXT:    lsl w15, w15, #8
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    mov v14.h[4], w11
; CHECK-GI-NEXT:    mov v29.h[3], w13
; CHECK-GI-NEXT:    ldr w11, [sp, #984]
; CHECK-GI-NEXT:    lsl w14, w14, #8
; CHECK-GI-NEXT:    sbfx w15, w15, #8, #8
; CHECK-GI-NEXT:    mov v13.h[5], w16
; CHECK-GI-NEXT:    ldr w16, [sp, #1040]
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #928]
; CHECK-GI-NEXT:    sbfx w14, w14, #8, #8
; CHECK-GI-NEXT:    mov v12.h[7], w10
; CHECK-GI-NEXT:    mov v27.s[3], w9
; CHECK-GI-NEXT:    mov v14.h[5], w12
; CHECK-GI-NEXT:    mov v29.h[4], w15
; CHECK-GI-NEXT:    lsl w16, w16, #8
; CHECK-GI-NEXT:    sbfx w10, w11, #8, #8
; CHECK-GI-NEXT:    lsl w13, w13, #8
; CHECK-GI-NEXT:    mov v13.h[6], w14
; CHECK-GI-NEXT:    ldr w12, [sp, #1048]
; CHECK-GI-NEXT:    sbfx w14, w16, #8, #8
; CHECK-GI-NEXT:    ldr w11, [sp, #728]
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    mul v15.8h, v8.8h, v12.8h
; CHECK-GI-NEXT:    smov w16, v31.h[6]
; CHECK-GI-NEXT:    mov v14.h[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #992]
; CHECK-GI-NEXT:    mov v29.h[5], w14
; CHECK-GI-NEXT:    lsl w12, w12, #8
; CHECK-GI-NEXT:    lsl w11, w11, #8
; CHECK-GI-NEXT:    mov v13.h[7], w13
; CHECK-GI-NEXT:    lsl w10, w10, #8
; CHECK-GI-NEXT:    ldr w13, [sp, #792]
; CHECK-GI-NEXT:    ldr w14, [sp, #1056]
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    mov v30.s[2], w16
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    smov w8, v15.h[1]
; CHECK-GI-NEXT:    smov w9, v15.h[5]
; CHECK-GI-NEXT:    mov v29.h[6], w12
; CHECK-GI-NEXT:    lsl w12, w13, #8
; CHECK-GI-NEXT:    lsl w13, w14, #8
; CHECK-GI-NEXT:    mov v10.h[7], w11
; CHECK-GI-NEXT:    mov v14.h[7], w10
; CHECK-GI-NEXT:    mul v12.8h, v9.8h, v13.8h
; CHECK-GI-NEXT:    sbfx w12, w12, #8, #8
; CHECK-GI-NEXT:    sbfx w13, w13, #8, #8
; CHECK-GI-NEXT:    smov w10, v15.h[0]
; CHECK-GI-NEXT:    smov w11, v15.h[4]
; CHECK-GI-NEXT:    smov w14, v31.h[7]
; CHECK-GI-NEXT:    smov w15, v31.h[3]
; CHECK-GI-NEXT:    mov v11.h[7], w12
; CHECK-GI-NEXT:    mov v29.h[7], w13
; CHECK-GI-NEXT:    mov v6.s[1], wzr
; CHECK-GI-NEXT:    mul v13.8h, v10.8h, v14.8h
; CHECK-GI-NEXT:    smov w12, v12.h[0]
; CHECK-GI-NEXT:    smov w13, v12.h[1]
; CHECK-GI-NEXT:    mov v7.s[1], wzr
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v4.s[1], wzr
; CHECK-GI-NEXT:    fmov s31, w11
; CHECK-GI-NEXT:    mov v30.s[3], w14
; CHECK-GI-NEXT:    smov w11, v12.h[4]
; CHECK-GI-NEXT:    mul v14.8h, v11.8h, v29.8h
; CHECK-GI-NEXT:    fmov s29, w10
; CHECK-GI-NEXT:    smov w10, v15.h[2]
; CHECK-GI-NEXT:    smov w14, v13.h[0]
; CHECK-GI-NEXT:    fmov s8, w12
; CHECK-GI-NEXT:    smov w16, v13.h[1]
; CHECK-GI-NEXT:    mov v31.s[1], w9
; CHECK-GI-NEXT:    smov w9, v12.h[2]
; CHECK-GI-NEXT:    mov v28.s[3], w15
; CHECK-GI-NEXT:    mov v29.s[1], w8
; CHECK-GI-NEXT:    smov w8, v15.h[6]
; CHECK-GI-NEXT:    smov w15, v12.h[5]
; CHECK-GI-NEXT:    mov v8.s[1], w13
; CHECK-GI-NEXT:    fmov s9, w11
; CHECK-GI-NEXT:    smov w12, v15.h[3]
; CHECK-GI-NEXT:    fmov s10, w14
; CHECK-GI-NEXT:    smov w14, v13.h[2]
; CHECK-GI-NEXT:    smov w11, v12.h[6]
; CHECK-GI-NEXT:    smov w13, v15.h[7]
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    mov v31.s[2], w8
; CHECK-GI-NEXT:    smov w8, v13.h[4]
; CHECK-GI-NEXT:    mov v29.s[2], w10
; CHECK-GI-NEXT:    mov v10.s[1], w16
; CHECK-GI-NEXT:    smov w16, v14.h[0]
; CHECK-GI-NEXT:    mov v8.s[2], w9
; CHECK-GI-NEXT:    smov w9, v13.h[5]
; CHECK-GI-NEXT:    smov w10, v12.h[3]
; CHECK-GI-NEXT:    mov v9.s[1], w15
; CHECK-GI-NEXT:    smov w15, v13.h[6]
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    fmov s11, w8
; CHECK-GI-NEXT:    smov w8, v14.h[1]
; CHECK-GI-NEXT:    mov v29.s[3], w12
; CHECK-GI-NEXT:    mov v10.s[2], w14
; CHECK-GI-NEXT:    smov w14, v12.h[7]
; CHECK-GI-NEXT:    fmov s12, w16
; CHECK-GI-NEXT:    smov w12, v14.h[4]
; CHECK-GI-NEXT:    mov v8.s[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #536]
; CHECK-GI-NEXT:    mov v11.s[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #272]
; CHECK-GI-NEXT:    mov v9.s[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #800]
; CHECK-GI-NEXT:    mov v12.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1064]
; CHECK-GI-NEXT:    mov v31.s[3], w13
; CHECK-GI-NEXT:    smov w13, v14.h[5]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v11.s[2], w15
; CHECK-GI-NEXT:    smov w15, v13.h[3]
; CHECK-GI-NEXT:    smov w16, v13.h[7]
; CHECK-GI-NEXT:    fmov s13, w12
; CHECK-GI-NEXT:    mul w9, w9, w10
; CHECK-GI-NEXT:    smov w12, v14.h[2]
; CHECK-GI-NEXT:    mul w8, w11, w8
; CHECK-GI-NEXT:    mov v19.s[2], wzr
; CHECK-GI-NEXT:    mov v21.s[2], wzr
; CHECK-GI-NEXT:    mov v16.s[2], wzr
; CHECK-GI-NEXT:    mov v18.s[2], wzr
; CHECK-GI-NEXT:    mov v17.s[2], wzr
; CHECK-GI-NEXT:    mov v13.s[1], w13
; CHECK-GI-NEXT:    smov w13, v14.h[6]
; CHECK-GI-NEXT:    sxth w9, w9
; CHECK-GI-NEXT:    sxth w10, w8
; CHECK-GI-NEXT:    mov v20.s[2], wzr
; CHECK-GI-NEXT:    mov v6.s[2], wzr
; CHECK-GI-NEXT:    mov v7.s[2], wzr
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    add v22.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    add v25.4s, v24.4s, v25.4s
; CHECK-GI-NEXT:    fmov s23, w9
; CHECK-GI-NEXT:    fmov s24, w10
; CHECK-GI-NEXT:    mov v12.s[2], w12
; CHECK-GI-NEXT:    mov v13.s[2], w13
; CHECK-GI-NEXT:    smov w8, v14.h[3]
; CHECK-GI-NEXT:    smov w9, v14.h[7]
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v19.s[3], wzr
; CHECK-GI-NEXT:    mov v21.s[3], wzr
; CHECK-GI-NEXT:    mov v16.s[3], wzr
; CHECK-GI-NEXT:    mov v18.s[3], wzr
; CHECK-GI-NEXT:    mov v17.s[3], wzr
; CHECK-GI-NEXT:    mov v20.s[3], wzr
; CHECK-GI-NEXT:    mov v6.s[3], wzr
; CHECK-GI-NEXT:    mov v7.s[3], wzr
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], wzr
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    mov v23.s[1], wzr
; CHECK-GI-NEXT:    mov v24.s[1], wzr
; CHECK-GI-NEXT:    mov v9.s[3], w14
; CHECK-GI-NEXT:    mov v10.s[3], w15
; CHECK-GI-NEXT:    mov v11.s[3], w16
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v12.s[3], w8
; CHECK-GI-NEXT:    mov v13.s[3], w9
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v19.4s, v19.4s, v21.4s
; CHECK-GI-NEXT:    add v16.4s, v16.4s, v18.4s
; CHECK-GI-NEXT:    add v17.4s, v17.4s, v20.4s
; CHECK-GI-NEXT:    add v6.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    add v3.4s, v3.4s, v5.4s
; CHECK-GI-NEXT:    mov v23.s[2], wzr
; CHECK-GI-NEXT:    mov v24.s[2], wzr
; CHECK-GI-NEXT:    add v26.4s, v26.4s, v27.4s
; CHECK-GI-NEXT:    add v27.4s, v28.4s, v30.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v19.4s
; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v5.4s, v29.4s, v31.4s
; CHECK-GI-NEXT:    add v7.4s, v8.4s, v9.4s
; CHECK-GI-NEXT:    add v16.4s, v10.4s, v11.4s
; CHECK-GI-NEXT:    add v17.4s, v12.4s, v13.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v6.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    mov v23.s[3], wzr
; CHECK-GI-NEXT:    mov v24.s[3], wzr
; CHECK-GI-NEXT:    add v3.4s, v22.4s, v25.4s
; CHECK-GI-NEXT:    add v6.4s, v26.4s, v27.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-GI-NEXT:    add v4.4s, v5.4s, v7.4s
; CHECK-GI-NEXT:    add v5.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    ldr x29, [sp, #64] // 8-byte Reload
; CHECK-GI-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v2.4s, v3.4s, v6.4s
; CHECK-GI-NEXT:    add v1.4s, v23.4s, v1.4s
; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    add v0.4s, v24.4s, v0.4s
; CHECK-GI-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <33 x i8> %a to <33 x i32>
  %bz = sext <33 x i8> %b to <33 x i32>
  %m1 = mul nuw nsw <33 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1)
  %cz = sext <33 x i8> %c to <33 x i32>
  %dz = sext <33 x i8> %d to <33 x i32>
  %m2 = mul nuw nsw <33 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v33i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    ldr b0, [sp, #80]
; CHECK-SD-NEXT:    add x8, sp, #88
; CHECK-SD-NEXT:    ldr b2, [sp, #144]
; CHECK-SD-NEXT:    add x9, sp, #152
; CHECK-SD-NEXT:    ldr b3, [sp, #16]
; CHECK-SD-NEXT:    add x11, sp, #104
; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #24
; CHECK-SD-NEXT:    add x8, sp, #96
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
; CHECK-SD-NEXT:    ldr b5, [sp, #480]
; CHECK-SD-NEXT:    fmov s1, w0
; CHECK-SD-NEXT:    add x10, sp, #112
; CHECK-SD-NEXT:    add x12, sp, #168
; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #160
; CHECK-SD-NEXT:    ldr b4, [sp, #608]
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    add x13, sp, #496
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
; CHECK-SD-NEXT:    mov v1.b[1], w1
; CHECK-SD-NEXT:    ldr b6, [sp, #672]
; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #488
; CHECK-SD-NEXT:    add x9, sp, #120
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
; CHECK-SD-NEXT:    add x11, sp, #40
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
; CHECK-SD-NEXT:    add x12, sp, #616
; CHECK-SD-NEXT:    ldr b16, [sp, #544]
; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #48
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
; CHECK-SD-NEXT:    add x12, sp, #176
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
; CHECK-SD-NEXT:    add x13, sp, #680
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]
; CHECK-SD-NEXT:    add x13, sp, #56
; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
; CHECK-SD-NEXT:    mov v1.b[2], w2
; CHECK-SD-NEXT:    add x8, sp, #128
; CHECK-SD-NEXT:    add x14, sp, #184
; CHECK-SD-NEXT:    add x11, sp, #136
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
; CHECK-SD-NEXT:    add x13, sp, #552
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x14]
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x13]
; CHECK-SD-NEXT:    add x14, sp, #624
; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
; CHECK-SD-NEXT:    add x8, sp, #688
; CHECK-SD-NEXT:    add x13, sp, #504
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x14]
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #560
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x8]
; CHECK-SD-NEXT:    mov v1.b[3], w3
; CHECK-SD-NEXT:    add x9, sp, #64
; CHECK-SD-NEXT:    add x15, sp, #632
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x15]
; CHECK-SD-NEXT:    add x8, sp, #696
; CHECK-SD-NEXT:    add x9, sp, #568
; CHECK-SD-NEXT:    add x11, sp, #512
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
; CHECK-SD-NEXT:    add x8, sp, #640
; CHECK-SD-NEXT:    mov v1.b[4], w4
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #704
; CHECK-SD-NEXT:    add x9, sp, #576
; CHECK-SD-NEXT:    add x11, sp, #520
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
; CHECK-SD-NEXT:    ldr b18, [sp, #736]
; CHECK-SD-NEXT:    add x12, sp, #192
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x12]
; CHECK-SD-NEXT:    add x8, sp, #648
; CHECK-SD-NEXT:    add x9, sp, #528
; CHECK-SD-NEXT:    add x11, sp, #712
; CHECK-SD-NEXT:    add x12, sp, #584
; CHECK-SD-NEXT:    sshll v18.8h, v18.8b, #0
; CHECK-SD-NEXT:    mov v1.b[5], w5
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x12]
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    add x8, sp, #656
; CHECK-SD-NEXT:    add x9, sp, #536
; CHECK-SD-NEXT:    add x11, sp, #720
; CHECK-SD-NEXT:    add x12, sp, #592
; CHECK-SD-NEXT:    sshll v18.4s, v18.4h, #0
; CHECK-SD-NEXT:    ldr b7, [sp, #208]
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
; CHECK-SD-NEXT:    mov v1.b[6], w6
; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
; CHECK-SD-NEXT:    add x8, sp, #664
; CHECK-SD-NEXT:    add x9, sp, #728
; CHECK-SD-NEXT:    add x11, sp, #600
; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
; CHECK-SD-NEXT:    add x10, sp, #200
; CHECK-SD-NEXT:    mov v1.b[7], w7
; CHECK-SD-NEXT:    add x9, sp, #72
; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
; CHECK-SD-NEXT:    saddw v17.4s, v17.4s, v5.4h
; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    mov v18.s[0], v7.s[0]
; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
; CHECK-SD-NEXT:    saddl2 v7.4s, v16.8h, v6.8h
; CHECK-SD-NEXT:    saddl2 v5.4s, v5.8h, v4.8h
; CHECK-SD-NEXT:    saddl v6.4s, v16.4h, v6.4h
; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v4.4h
; CHECK-SD-NEXT:    saddl2 v17.4s, v1.8h, v0.8h
; CHECK-SD-NEXT:    saddl2 v16.4s, v3.8h, v2.8h
; CHECK-SD-NEXT:    saddw v1.4s, v18.4s, v1.4h
; CHECK-SD-NEXT:    add v5.4s, v5.4s, v7.4s
; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
; CHECK-SD-NEXT:    saddl v2.4s, v3.4h, v2.4h
; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT:    add v1.4s, v4.4s, v5.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v33i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    stp d13, d12, [sp, #-64]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    str x29, [sp, #48] // 8-byte Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    .cfi_offset b8, -24
; CHECK-GI-NEXT:    .cfi_offset b9, -32
; CHECK-GI-NEXT:    .cfi_offset b10, -40
; CHECK-GI-NEXT:    .cfi_offset b11, -48
; CHECK-GI-NEXT:    .cfi_offset b12, -56
; CHECK-GI-NEXT:    .cfi_offset b13, -64
; CHECK-GI-NEXT:    sxtb w8, w0
; CHECK-GI-NEXT:    sxtb w9, w1
; CHECK-GI-NEXT:    sxtb w12, w4
; CHECK-GI-NEXT:    ldr w10, [sp, #72]
; CHECK-GI-NEXT:    ldr w11, [sp, #96]
; CHECK-GI-NEXT:    sxtb w13, w5
; CHECK-GI-NEXT:    fmov s22, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #64]
; CHECK-GI-NEXT:    fmov s23, w12
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w12, w7
; CHECK-GI-NEXT:    movi d18, #0000000000000000
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    movi d19, #0000000000000000
; CHECK-GI-NEXT:    movi d20, #0000000000000000
; CHECK-GI-NEXT:    mov v22.s[1], w9
; CHECK-GI-NEXT:    sxtb w9, w2
; CHECK-GI-NEXT:    mov v23.s[1], w13
; CHECK-GI-NEXT:    fmov s24, w8
; CHECK-GI-NEXT:    sxtb w8, w11
; CHECK-GI-NEXT:    sxtb w11, w6
; CHECK-GI-NEXT:    ldr w13, [sp, #232]
; CHECK-GI-NEXT:    mov v18.s[1], wzr
; CHECK-GI-NEXT:    movi d21, #0000000000000000
; CHECK-GI-NEXT:    fmov s25, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #80]
; CHECK-GI-NEXT:    mov v19.s[1], wzr
; CHECK-GI-NEXT:    mov v22.s[2], w9
; CHECK-GI-NEXT:    mov v24.s[1], w10
; CHECK-GI-NEXT:    sxtb w10, w3
; CHECK-GI-NEXT:    ldr w9, [sp, #104]
; CHECK-GI-NEXT:    mov v23.s[2], w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    ldr w11, [sp, #136]
; CHECK-GI-NEXT:    mov v18.s[2], wzr
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v19.s[2], wzr
; CHECK-GI-NEXT:    movi d7, #0000000000000000
; CHECK-GI-NEXT:    mov v22.s[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #128]
; CHECK-GI-NEXT:    mov v24.s[2], w8
; CHECK-GI-NEXT:    mov v25.s[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #112]
; CHECK-GI-NEXT:    ldr w8, [sp, #88]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    mov v23.s[3], w12
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    ldr w12, [sp, #160]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    fmov s26, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #144]
; CHECK-GI-NEXT:    movi d5, #0000000000000000
; CHECK-GI-NEXT:    mov v25.s[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #120]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v24.s[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #168]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v26.s[1], w11
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    fmov s27, w12
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    ldr w12, [sp, #224]
; CHECK-GI-NEXT:    ldr w11, [sp, #152]
; CHECK-GI-NEXT:    mov v25.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #192]
; CHECK-GI-NEXT:    add v22.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    mov v27.s[1], w8
; CHECK-GI-NEXT:    movi d16, #0000000000000000
; CHECK-GI-NEXT:    movi d17, #0000000000000000
; CHECK-GI-NEXT:    mov v26.s[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #200]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    movi d3, #0000000000000000
; CHECK-GI-NEXT:    sxtb w8, w10
; CHECK-GI-NEXT:    sxtb w10, w12
; CHECK-GI-NEXT:    fmov s28, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #176]
; CHECK-GI-NEXT:    sxtb w12, w13
; CHECK-GI-NEXT:    ldr w13, [sp, #208]
; CHECK-GI-NEXT:    fmov s29, w10
; CHECK-GI-NEXT:    sxtb w10, w11
; CHECK-GI-NEXT:    ldr w11, [sp, #536]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v28.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #184]
; CHECK-GI-NEXT:    mov v26.s[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #528]
; CHECK-GI-NEXT:    sxtb w13, w13
; CHECK-GI-NEXT:    mov v29.s[1], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #240]
; CHECK-GI-NEXT:    mov v27.s[2], w9
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    ldr w9, [sp, #216]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v28.s[2], w13
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    fmov s30, w10
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    ldr w10, [sp, #256]
; CHECK-GI-NEXT:    mov v29.s[2], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #248]
; CHECK-GI-NEXT:    mov v27.s[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #560]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    add v23.4s, v24.4s, v25.4s
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v30.s[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #544]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v28.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #568]
; CHECK-GI-NEXT:    mov v29.s[3], w12
; CHECK-GI-NEXT:    ldr w12, [sp, #592]
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    fmov s31, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    ldr w8, [sp, #552]
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    mov v30.s[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #600]
; CHECK-GI-NEXT:    fmov s8, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #624]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v31.s[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #576]
; CHECK-GI-NEXT:    fmov s9, w12
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    movi d2, #0000000000000000
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v30.s[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #632]
; CHECK-GI-NEXT:    mov v9.s[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #608]
; CHECK-GI-NEXT:    fmov s10, w10
; CHECK-GI-NEXT:    mov v31.s[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #656]
; CHECK-GI-NEXT:    ldr w10, [sp, #688]
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    movi d4, #0000000000000000
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v18.s[3], wzr
; CHECK-GI-NEXT:    mov v9.s[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #664]
; CHECK-GI-NEXT:    mov v10.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #696]
; CHECK-GI-NEXT:    fmov s11, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #640]
; CHECK-GI-NEXT:    fmov s12, w10
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    ldr w10, [sp, #672]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v19.s[3], wzr
; CHECK-GI-NEXT:    mov v11.s[1], w11
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    mov v20.s[1], wzr
; CHECK-GI-NEXT:    mov v12.s[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #704]
; CHECK-GI-NEXT:    mov v10.s[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #648]
; CHECK-GI-NEXT:    mov v21.s[1], wzr
; CHECK-GI-NEXT:    mov v6.s[1], wzr
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v7.s[1], wzr
; CHECK-GI-NEXT:    mov v5.s[1], wzr
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v11.s[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #680]
; CHECK-GI-NEXT:    mov v12.s[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #712]
; CHECK-GI-NEXT:    mov v16.s[1], wzr
; CHECK-GI-NEXT:    mov v10.s[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #720]
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v17.s[1], wzr
; CHECK-GI-NEXT:    add v19.4s, v18.4s, v19.4s
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v11.s[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #728]
; CHECK-GI-NEXT:    mov v12.s[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #752]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    fmov s13, w9
; CHECK-GI-NEXT:    sxtb w10, w10
; CHECK-GI-NEXT:    ldr w9, [sp, #760]
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    mov v4.s[1], wzr
; CHECK-GI-NEXT:    mov v13.s[1], w10
; CHECK-GI-NEXT:    fmov s24, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #736]
; CHECK-GI-NEXT:    mov v20.s[2], wzr
; CHECK-GI-NEXT:    mov v21.s[2], wzr
; CHECK-GI-NEXT:    mov v6.s[2], wzr
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    mov v7.s[2], wzr
; CHECK-GI-NEXT:    mov v8.s[1], wzr
; CHECK-GI-NEXT:    mov v24.s[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #768]
; CHECK-GI-NEXT:    ldr w12, [sp, #584]
; CHECK-GI-NEXT:    mov v13.s[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #784]
; CHECK-GI-NEXT:    ldr w11, [sp, #616]
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v5.s[2], wzr
; CHECK-GI-NEXT:    mov v16.s[2], wzr
; CHECK-GI-NEXT:    sxtb w10, w8
; CHECK-GI-NEXT:    ldr w8, [sp, #744]
; CHECK-GI-NEXT:    mov v17.s[2], wzr
; CHECK-GI-NEXT:    mov v24.s[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #776]
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    fmov s18, w10
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v4.s[2], wzr
; CHECK-GI-NEXT:    mov v20.s[3], wzr
; CHECK-GI-NEXT:    mov v21.s[3], wzr
; CHECK-GI-NEXT:    mov v6.s[3], wzr
; CHECK-GI-NEXT:    mov v7.s[3], wzr
; CHECK-GI-NEXT:    mov v18.s[1], wzr
; CHECK-GI-NEXT:    sxtb w12, w12
; CHECK-GI-NEXT:    sxtb w11, w11
; CHECK-GI-NEXT:    sxtb w8, w8
; CHECK-GI-NEXT:    sxtb w9, w9
; CHECK-GI-NEXT:    mov v8.s[2], wzr
; CHECK-GI-NEXT:    mov v31.s[3], w12
; CHECK-GI-NEXT:    mov v9.s[3], w11
; CHECK-GI-NEXT:    mov v5.s[3], wzr
; CHECK-GI-NEXT:    mov v16.s[3], wzr
; CHECK-GI-NEXT:    mov v17.s[3], wzr
; CHECK-GI-NEXT:    mov v13.s[3], w8
; CHECK-GI-NEXT:    mov v18.s[2], wzr
; CHECK-GI-NEXT:    mov v24.s[3], w9
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v3.s[3], wzr
; CHECK-GI-NEXT:    mov v2.s[3], wzr
; CHECK-GI-NEXT:    mov v4.s[3], wzr
; CHECK-GI-NEXT:    add v20.4s, v20.4s, v21.4s
; CHECK-GI-NEXT:    add v6.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    mov v8.s[3], wzr
; CHECK-GI-NEXT:    add v25.4s, v26.4s, v27.4s
; CHECK-GI-NEXT:    add v26.4s, v28.4s, v29.4s
; CHECK-GI-NEXT:    mov v18.s[3], wzr
; CHECK-GI-NEXT:    add v16.4s, v16.4s, v17.4s
; CHECK-GI-NEXT:    add v7.4s, v30.4s, v31.4s
; CHECK-GI-NEXT:    add v5.4s, v5.4s, v20.4s
; CHECK-GI-NEXT:    add v17.4s, v9.4s, v10.4s
; CHECK-GI-NEXT:    add v20.4s, v11.4s, v12.4s
; CHECK-GI-NEXT:    add v21.4s, v13.4s, v24.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v6.4s
; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    add v4.4s, v25.4s, v26.4s
; CHECK-GI-NEXT:    add v5.4s, v8.4s, v5.4s
; CHECK-GI-NEXT:    add v6.4s, v19.4s, v16.4s
; CHECK-GI-NEXT:    add v7.4s, v7.4s, v17.4s
; CHECK-GI-NEXT:    add v16.4s, v20.4s, v21.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    ldr x29, [sp, #48] // 8-byte Reload
; CHECK-GI-NEXT:    add v0.4s, v18.4s, v0.4s
; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v4.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldp d13, d12, [sp], #64 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <33 x i8> %a to <33 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)
  %cz = sext <33 x i8> %c to <33 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v48i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q1, [x0, #32]
; CHECK-SD-NEXT:    ldr q2, [x1, #32]
; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    ldp q3, q1, [x0]
; CHECK-SD-NEXT:    ldp q4, q2, [x1]
; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v48i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q7, [x0, #32]
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q17, [x1, #32]
; CHECK-GI-NEXT:    ldp q4, q5, [x0]
; CHECK-GI-NEXT:    ldp q6, q16, [x1]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    udot v2.4s, v17.16b, v7.16b
; CHECK-GI-NEXT:    udot v1.4s, v6.16b, v4.16b
; CHECK-GI-NEXT:    udot v3.4s, v16.16b, v5.16b
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <48 x i8>, ptr %a
  %1 = zext <48 x i8> %0 to <48 x i32>
  %2 = load <48 x i8>, ptr %b
  %3 = zext <48 x i8> %2 to <48 x i32>
  %4 = mul nuw nsw <48 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v48i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.16b, #1
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q2, [x0, #32]
; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
; CHECK-SD-NEXT:    ldp q3, q2, [x0]
; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
; CHECK-SD-NEXT:    addv s0, v1.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v48i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi v1.16b, #1
; CHECK-GI-NEXT:    ldr q7, [x0, #32]
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q5, q6, [x0]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v1.16b
; CHECK-GI-NEXT:    udot v4.4s, v6.16b, v1.16b
; CHECK-GI-NEXT:    udot v3.4s, v7.16b, v1.16b
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <48 x i8>, ptr %a1
  %1 = zext <48 x i8> %0 to <48 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v48i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    ldr q1, [x0, #32]
; CHECK-SD-NEXT:    ldr q2, [x1, #32]
; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    ldp q3, q1, [x0]
; CHECK-SD-NEXT:    ldp q4, q2, [x1]
; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b
; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v48i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q7, [x0, #32]
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    ldr q17, [x1, #32]
; CHECK-GI-NEXT:    ldp q4, q5, [x0]
; CHECK-GI-NEXT:    ldp q6, q16, [x1]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    sdot v2.4s, v17.16b, v7.16b
; CHECK-GI-NEXT:    sdot v1.4s, v6.16b, v4.16b
; CHECK-GI-NEXT:    sdot v3.4s, v16.16b, v5.16b
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <48 x i8>, ptr %a
  %1 = sext <48 x i8> %0 to <48 x i32>
  %2 = load <48 x i8>, ptr %b
  %3 = sext <48 x i8> %2 to <48 x i32>
  %4 = mul nsw <48 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v48i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    ldr b3, [sp, #592]
; CHECK-SD-NEXT:    add x8, sp, #600
; CHECK-SD-NEXT:    ldr b6, [sp, #208]
; CHECK-SD-NEXT:    ldr b0, [sp, #336]
; CHECK-SD-NEXT:    add x9, sp, #344
; CHECK-SD-NEXT:    ldr b2, [sp, #464]
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #216
; CHECK-SD-NEXT:    add x10, sp, #624
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #608
; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #232
; CHECK-SD-NEXT:    fmov s1, w0
; CHECK-SD-NEXT:    ldr b7, [sp, #1360]
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #224
; CHECK-SD-NEXT:    add x11, sp, #648
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #616
; CHECK-SD-NEXT:    add x12, sp, #376
; CHECK-SD-NEXT:    mov v1.b[1], w1
; CHECK-SD-NEXT:    ldr b16, [sp, #976]
; CHECK-SD-NEXT:    add x14, sp, #288
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #632
; CHECK-SD-NEXT:    add x15, sp, #408
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #472
; CHECK-SD-NEXT:    add x13, sp, #696
; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #240
; CHECK-SD-NEXT:    add x16, sp, #448
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #352
; CHECK-SD-NEXT:    mov v1.b[2], w2
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #1368
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
; CHECK-SD-NEXT:    add x10, sp, #248
; CHECK-SD-NEXT:    add x9, sp, #640
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #656
; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
; CHECK-SD-NEXT:    add x10, sp, #360
; CHECK-SD-NEXT:    mov v1.b[3], w3
; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x10]
; CHECK-SD-NEXT:    add x10, sp, #256
; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #368
; CHECK-SD-NEXT:    ldr b17, [sp, #720]
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]
; CHECK-SD-NEXT:    add x10, sp, #984
; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]
; CHECK-SD-NEXT:    add x10, sp, #664
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x11]
; CHECK-SD-NEXT:    add x11, sp, #264
; CHECK-SD-NEXT:    mov v1.b[4], w4
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x11]
; CHECK-SD-NEXT:    add x9, sp, #672
; CHECK-SD-NEXT:    add x11, sp, #680
; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x12]
; CHECK-SD-NEXT:    add x12, sp, #480
; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]
; CHECK-SD-NEXT:    add x12, sp, #272
; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x8]
; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x12]
; CHECK-SD-NEXT:    add x12, sp, #384
; CHECK-SD-NEXT:    mov v1.b[5], w5
; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x12]
; CHECK-SD-NEXT:    add x12, sp, #280
; CHECK-SD-NEXT:    add x8, sp, #688
; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x10]
; CHECK-SD-NEXT:    add x10, sp, #1376
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #392
; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x12]
; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x10]
; CHECK-SD-NEXT:    mov v1.b[6], w6
; CHECK-SD-NEXT:    add x12, sp, #704
; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]
; CHECK-SD-NEXT:    add x9, sp, #400
; CHECK-SD-NEXT:    add x10, sp, #712
; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x14]
; CHECK-SD-NEXT:    add x14, sp, #992
; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x14]
; CHECK-SD-NEXT:    add x14, sp, #296
; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x11]
; CHECK-SD-NEXT:    add x9, sp, #304
; CHECK-SD-NEXT:    add x11, sp, #312
; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x14]
; CHECK-SD-NEXT:    mov v1.b[7], w7
; CHECK-SD-NEXT:    add x14, sp, #320
; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x15]
; CHECK-SD-NEXT:    add x15, sp, #328
; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x8]
; CHECK-SD-NEXT:    add x8, sp, #416
; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1384
; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #424
; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x13]
; CHECK-SD-NEXT:    add x8, sp, #432
; CHECK-SD-NEXT:    add x13, sp, #440
; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x11]
; CHECK-SD-NEXT:    add x11, sp, #16
; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1000
; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x12]
; CHECK-SD-NEXT:    add x12, sp, #488
; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x14]
; CHECK-SD-NEXT:    add x14, sp, #1392
; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x14]
; CHECK-SD-NEXT:    add x11, sp, #1008
; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1400
; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x10]
; CHECK-SD-NEXT:    add x10, sp, #496
; CHECK-SD-NEXT:    add x9, sp, #24
; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x15]
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #1016
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]
; CHECK-SD-NEXT:    add x8, sp, #1408
; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x9]
; CHECK-SD-NEXT:    add x9, sp, #504
; CHECK-SD-NEXT:    add x10, sp, #512
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1024
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x16]
; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1416
; CHECK-SD-NEXT:    add x9, sp, #456
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
; CHECK-SD-NEXT:    add x10, sp, #1032
; CHECK-SD-NEXT:    add x8, sp, #40
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x9]
; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1424
; CHECK-SD-NEXT:    add x9, sp, #520
; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1040
; CHECK-SD-NEXT:    add x8, sp, #48
; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x9]
; CHECK-SD-NEXT:    add x10, sp, #528
; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1432
; CHECK-SD-NEXT:    sdot v5.4s, v6.16b, v3.16b
; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x8]
; CHECK-SD-NEXT:    ld1 { v2.b }[8], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1048
; CHECK-SD-NEXT:    ldr b3, [sp, #80]
; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x8]
; CHECK-SD-NEXT:    add x10, sp, #88
; CHECK-SD-NEXT:    add x8, sp, #536
; CHECK-SD-NEXT:    add x11, sp, #1440
; CHECK-SD-NEXT:    add x9, sp, #56
; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v2.b }[9], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1056
; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x8]
; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x9]
; CHECK-SD-NEXT:    add x9, sp, #96
; CHECK-SD-NEXT:    add x8, sp, #544
; CHECK-SD-NEXT:    add x10, sp, #1448
; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v2.b }[10], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1064
; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x8]
; CHECK-SD-NEXT:    add x10, sp, #104
; CHECK-SD-NEXT:    add x8, sp, #552
; CHECK-SD-NEXT:    add x11, sp, #1456
; CHECK-SD-NEXT:    add x9, sp, #64
; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
; CHECK-SD-NEXT:    ld1 { v2.b }[11], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1072
; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x8]
; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]
; CHECK-SD-NEXT:    add x9, sp, #112
; CHECK-SD-NEXT:    add x8, sp, #560
; CHECK-SD-NEXT:    add x10, sp, #1464
; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v2.b }[12], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1080
; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x8]
; CHECK-SD-NEXT:    add x10, sp, #120
; CHECK-SD-NEXT:    add x8, sp, #568
; CHECK-SD-NEXT:    add x11, sp, #1472
; CHECK-SD-NEXT:    add x9, sp, #72
; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v2.b }[13], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1088
; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x11]
; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x8]
; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x9]
; CHECK-SD-NEXT:    add x9, sp, #128
; CHECK-SD-NEXT:    ldr b6, [sp, #1104]
; CHECK-SD-NEXT:    add x10, sp, #1480
; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
; CHECK-SD-NEXT:    add x8, sp, #1096
; CHECK-SD-NEXT:    add x9, sp, #1112
; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x8]
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]
; CHECK-SD-NEXT:    add x8, sp, #728
; CHECK-SD-NEXT:    add x9, sp, #576
; CHECK-SD-NEXT:    add x10, sp, #136
; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1120
; CHECK-SD-NEXT:    ld1 { v2.b }[14], [x9]
; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #736
; CHECK-SD-NEXT:    ldr b7, [sp, #1232]
; CHECK-SD-NEXT:    ldr b16, [sp, #848]
; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x10]
; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x8]
; CHECK-SD-NEXT:    add x9, sp, #1240
; CHECK-SD-NEXT:    add x10, sp, #856
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1128
; CHECK-SD-NEXT:    add x11, sp, #744
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1248
; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
; CHECK-SD-NEXT:    add x11, sp, #864
; CHECK-SD-NEXT:    add x9, sp, #144
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1136
; CHECK-SD-NEXT:    add x12, sp, #752
; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1256
; CHECK-SD-NEXT:    add x10, sp, #872
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1144
; CHECK-SD-NEXT:    add x11, sp, #760
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1264
; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x11]
; CHECK-SD-NEXT:    add x11, sp, #880
; CHECK-SD-NEXT:    add x9, sp, #152
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1152
; CHECK-SD-NEXT:    add x12, sp, #768
; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1272
; CHECK-SD-NEXT:    add x10, sp, #888
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1160
; CHECK-SD-NEXT:    add x11, sp, #776
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1280
; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x11]
; CHECK-SD-NEXT:    add x11, sp, #896
; CHECK-SD-NEXT:    add x9, sp, #160
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1168
; CHECK-SD-NEXT:    add x12, sp, #784
; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[8], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1288
; CHECK-SD-NEXT:    add x10, sp, #904
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1176
; CHECK-SD-NEXT:    add x11, sp, #792
; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1296
; CHECK-SD-NEXT:    ld1 { v17.b }[9], [x11]
; CHECK-SD-NEXT:    add x11, sp, #912
; CHECK-SD-NEXT:    add x9, sp, #168
; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1184
; CHECK-SD-NEXT:    add x12, sp, #800
; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[10], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1304
; CHECK-SD-NEXT:    add x10, sp, #920
; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1192
; CHECK-SD-NEXT:    add x11, sp, #808
; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1312
; CHECK-SD-NEXT:    ld1 { v17.b }[11], [x11]
; CHECK-SD-NEXT:    add x11, sp, #928
; CHECK-SD-NEXT:    add x9, sp, #176
; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1200
; CHECK-SD-NEXT:    add x12, sp, #816
; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[12], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1320
; CHECK-SD-NEXT:    add x10, sp, #936
; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1208
; CHECK-SD-NEXT:    add x11, sp, #824
; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x8]
; CHECK-SD-NEXT:    add x10, sp, #1328
; CHECK-SD-NEXT:    ld1 { v17.b }[13], [x11]
; CHECK-SD-NEXT:    add x11, sp, #944
; CHECK-SD-NEXT:    add x9, sp, #184
; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x11]
; CHECK-SD-NEXT:    add x8, sp, #1216
; CHECK-SD-NEXT:    add x12, sp, #832
; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x9]
; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x8]
; CHECK-SD-NEXT:    ld1 { v17.b }[14], [x12]
; CHECK-SD-NEXT:    add x9, sp, #1336
; CHECK-SD-NEXT:    add x10, sp, #952
; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x10]
; CHECK-SD-NEXT:    add x8, sp, #1224
; CHECK-SD-NEXT:    add x11, sp, #840
; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x8]
; CHECK-SD-NEXT:    add x8, sp, #192
; CHECK-SD-NEXT:    ld1 { v17.b }[15], [x11]
; CHECK-SD-NEXT:    add x10, sp, #1344
; CHECK-SD-NEXT:    add x11, sp, #960
; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x11]
; CHECK-SD-NEXT:    add x9, sp, #584
; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v0.16b
; CHECK-SD-NEXT:    add x8, sp, #200
; CHECK-SD-NEXT:    sdot v4.4s, v17.16b, v6.16b
; CHECK-SD-NEXT:    ld1 { v2.b }[15], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1352
; CHECK-SD-NEXT:    add x10, sp, #968
; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x8]
; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x10]
; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v2.16b
; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b
; CHECK-SD-NEXT:    add v0.4s, v5.4s, v4.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v48i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    ldr w10, [sp, #80]
; CHECK-GI-NEXT:    ldr w9, [sp, #88]
; CHECK-GI-NEXT:    fmov s2, w0
; CHECK-GI-NEXT:    ldr w11, [sp, #208]
; CHECK-GI-NEXT:    ldr w8, [sp, #216]
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    fmov s3, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #336]
; CHECK-GI-NEXT:    ldr w12, [sp, #720]
; CHECK-GI-NEXT:    fmov s4, w11
; CHECK-GI-NEXT:    mov v2.b[1], w1
; CHECK-GI-NEXT:    ldr w11, [sp, #344]
; CHECK-GI-NEXT:    fmov s5, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #592]
; CHECK-GI-NEXT:    fmov s16, w12
; CHECK-GI-NEXT:    mov v3.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #464]
; CHECK-GI-NEXT:    ldr w12, [sp, #1112]
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    mov v4.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #472]
; CHECK-GI-NEXT:    fmov s6, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #600]
; CHECK-GI-NEXT:    mov v5.b[1], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #224]
; CHECK-GI-NEXT:    mov v2.b[2], w2
; CHECK-GI-NEXT:    ldr w11, [sp, #16]
; CHECK-GI-NEXT:    mov v7.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #480]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    mov v6.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #96]
; CHECK-GI-NEXT:    mov v4.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #608]
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v3.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #352]
; CHECK-GI-NEXT:    mov v2.b[3], w3
; CHECK-GI-NEXT:    mov v7.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #488]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v5.b[2], w8
; CHECK-GI-NEXT:    mov v6.b[2], w9
; CHECK-GI-NEXT:    ldr w8, [sp, #104]
; CHECK-GI-NEXT:    ldr w9, [sp, #232]
; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v3.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #360]
; CHECK-GI-NEXT:    mov v2.b[4], w4
; CHECK-GI-NEXT:    mov v4.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #616]
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v5.b[3], w8
; CHECK-GI-NEXT:    mov v6.b[3], w10
; CHECK-GI-NEXT:    ldr w8, [sp, #112]
; CHECK-GI-NEXT:    ldr w10, [sp, #240]
; CHECK-GI-NEXT:    mov v7.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #496]
; CHECK-GI-NEXT:    mov v3.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #368]
; CHECK-GI-NEXT:    mov v2.b[5], w5
; CHECK-GI-NEXT:    mov v4.b[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #624]
; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v5.b[4], w8
; CHECK-GI-NEXT:    mov v6.b[4], w9
; CHECK-GI-NEXT:    ldr w8, [sp, #120]
; CHECK-GI-NEXT:    ldr w9, [sp, #248]
; CHECK-GI-NEXT:    mov v7.b[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #504]
; CHECK-GI-NEXT:    mov v3.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #376]
; CHECK-GI-NEXT:    mov v2.b[6], w6
; CHECK-GI-NEXT:    mov v4.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #632]
; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v5.b[5], w8
; CHECK-GI-NEXT:    mov v6.b[5], w10
; CHECK-GI-NEXT:    ldr w8, [sp, #128]
; CHECK-GI-NEXT:    ldr w10, [sp, #256]
; CHECK-GI-NEXT:    mov v7.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #512]
; CHECK-GI-NEXT:    mov v3.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #384]
; CHECK-GI-NEXT:    mov v2.b[7], w7
; CHECK-GI-NEXT:    mov v4.b[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #640]
; CHECK-GI-NEXT:    movi v27.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v5.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #136]
; CHECK-GI-NEXT:    mov v6.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #264]
; CHECK-GI-NEXT:    mov v7.b[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #520]
; CHECK-GI-NEXT:    mov v3.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #392]
; CHECK-GI-NEXT:    mov v2.b[8], w11
; CHECK-GI-NEXT:    mov v4.b[7], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #648]
; CHECK-GI-NEXT:    ldr w11, [sp, #24]
; CHECK-GI-NEXT:    mov v5.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #144]
; CHECK-GI-NEXT:    mov v6.b[7], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #272]
; CHECK-GI-NEXT:    mov v7.b[7], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #528]
; CHECK-GI-NEXT:    mov v3.b[8], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #400]
; CHECK-GI-NEXT:    mov v2.b[9], w11
; CHECK-GI-NEXT:    mov v4.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #656]
; CHECK-GI-NEXT:    ldr w11, [sp, #32]
; CHECK-GI-NEXT:    mov v5.b[8], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #152]
; CHECK-GI-NEXT:    mov v6.b[8], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #280]
; CHECK-GI-NEXT:    mov v7.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #536]
; CHECK-GI-NEXT:    mov v3.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #408]
; CHECK-GI-NEXT:    mov v2.b[10], w11
; CHECK-GI-NEXT:    mov v4.b[9], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #664]
; CHECK-GI-NEXT:    ldr w11, [sp, #40]
; CHECK-GI-NEXT:    mov v5.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #160]
; CHECK-GI-NEXT:    mov v6.b[9], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #288]
; CHECK-GI-NEXT:    mov v7.b[9], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #544]
; CHECK-GI-NEXT:    mov v3.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #416]
; CHECK-GI-NEXT:    mov v2.b[11], w11
; CHECK-GI-NEXT:    mov v4.b[10], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #672]
; CHECK-GI-NEXT:    ldr w11, [sp, #48]
; CHECK-GI-NEXT:    mov v5.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #168]
; CHECK-GI-NEXT:    mov v6.b[10], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #296]
; CHECK-GI-NEXT:    mov v7.b[10], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #552]
; CHECK-GI-NEXT:    mov v3.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #424]
; CHECK-GI-NEXT:    mov v2.b[12], w11
; CHECK-GI-NEXT:    mov v4.b[11], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #680]
; CHECK-GI-NEXT:    ldr w11, [sp, #56]
; CHECK-GI-NEXT:    mov v5.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #176]
; CHECK-GI-NEXT:    mov v6.b[11], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #304]
; CHECK-GI-NEXT:    mov v7.b[11], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #560]
; CHECK-GI-NEXT:    mov v3.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #432]
; CHECK-GI-NEXT:    mov v2.b[13], w11
; CHECK-GI-NEXT:    mov v4.b[12], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #688]
; CHECK-GI-NEXT:    ldr w11, [sp, #64]
; CHECK-GI-NEXT:    mov v5.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #184]
; CHECK-GI-NEXT:    mov v6.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #312]
; CHECK-GI-NEXT:    mov v7.b[12], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #568]
; CHECK-GI-NEXT:    mov v3.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #440]
; CHECK-GI-NEXT:    mov v2.b[14], w11
; CHECK-GI-NEXT:    mov v4.b[13], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #696]
; CHECK-GI-NEXT:    ldr w11, [sp, #72]
; CHECK-GI-NEXT:    mov v5.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #192]
; CHECK-GI-NEXT:    mov v6.b[13], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #320]
; CHECK-GI-NEXT:    mov v7.b[13], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #576]
; CHECK-GI-NEXT:    mov v3.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #448]
; CHECK-GI-NEXT:    mov v2.b[15], w11
; CHECK-GI-NEXT:    mov v4.b[14], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #704]
; CHECK-GI-NEXT:    ldr w11, [sp, #848]
; CHECK-GI-NEXT:    mov v5.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #200]
; CHECK-GI-NEXT:    mov v6.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #328]
; CHECK-GI-NEXT:    mov v7.b[14], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #584]
; CHECK-GI-NEXT:    mov v3.b[15], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #456]
; CHECK-GI-NEXT:    fmov s17, w11
; CHECK-GI-NEXT:    mov v4.b[15], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #712]
; CHECK-GI-NEXT:    ldr w11, [sp, #1104]
; CHECK-GI-NEXT:    mov v5.b[15], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #856]
; CHECK-GI-NEXT:    mov v6.b[15], w10
; CHECK-GI-NEXT:    mov v7.b[15], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #976]
; CHECK-GI-NEXT:    ldr w10, [sp, #728]
; CHECK-GI-NEXT:    mov v17.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1232]
; CHECK-GI-NEXT:    fmov s19, w11
; CHECK-GI-NEXT:    fmov s18, w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1360]
; CHECK-GI-NEXT:    mov v16.b[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #984]
; CHECK-GI-NEXT:    fmov s20, w8
; CHECK-GI-NEXT:    ldr w11, [sp, #1240]
; CHECK-GI-NEXT:    fmov s21, w9
; CHECK-GI-NEXT:    ldr w8, [sp, #1368]
; CHECK-GI-NEXT:    mov v19.b[1], w12
; CHECK-GI-NEXT:    mov v18.b[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #736]
; CHECK-GI-NEXT:    ldr w9, [sp, #992]
; CHECK-GI-NEXT:    mov v20.b[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #864]
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    mov v21.b[1], w8
; CHECK-GI-NEXT:    mov v16.b[2], w10
; CHECK-GI-NEXT:    ldr w8, [sp, #1120]
; CHECK-GI-NEXT:    mov v17.b[2], w11
; CHECK-GI-NEXT:    ldr w10, [sp, #1248]
; CHECK-GI-NEXT:    ldr w11, [sp, #744]
; CHECK-GI-NEXT:    mov v18.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1376]
; CHECK-GI-NEXT:    mov v19.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #872]
; CHECK-GI-NEXT:    mov v20.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1000]
; CHECK-GI-NEXT:    mov v21.b[2], w9
; CHECK-GI-NEXT:    mov v16.b[3], w11
; CHECK-GI-NEXT:    ldr w9, [sp, #1256]
; CHECK-GI-NEXT:    mov v17.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1128]
; CHECK-GI-NEXT:    ldr w11, [sp, #752]
; CHECK-GI-NEXT:    mov v18.b[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1384]
; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v5.16b
; CHECK-GI-NEXT:    mov v19.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #880]
; CHECK-GI-NEXT:    mov v20.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1008]
; CHECK-GI-NEXT:    mov v21.b[3], w10
; CHECK-GI-NEXT:    mov v16.b[4], w11
; CHECK-GI-NEXT:    mov v17.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1136]
; CHECK-GI-NEXT:    ldr w10, [sp, #1264]
; CHECK-GI-NEXT:    mov v18.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1392]
; CHECK-GI-NEXT:    ldr w11, [sp, #760]
; CHECK-GI-NEXT:    mov v19.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #888]
; CHECK-GI-NEXT:    mov v20.b[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1016]
; CHECK-GI-NEXT:    mov v21.b[4], w9
; CHECK-GI-NEXT:    mov v16.b[5], w11
; CHECK-GI-NEXT:    mov v17.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1144]
; CHECK-GI-NEXT:    ldr w9, [sp, #1272]
; CHECK-GI-NEXT:    mov v18.b[5], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1400]
; CHECK-GI-NEXT:    ldr w11, [sp, #768]
; CHECK-GI-NEXT:    mov v19.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #896]
; CHECK-GI-NEXT:    mov v20.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1024]
; CHECK-GI-NEXT:    mov v21.b[5], w10
; CHECK-GI-NEXT:    mov v16.b[6], w11
; CHECK-GI-NEXT:    mov v17.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1152]
; CHECK-GI-NEXT:    ldr w10, [sp, #1280]
; CHECK-GI-NEXT:    mov v18.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1408]
; CHECK-GI-NEXT:    ldr w11, [sp, #776]
; CHECK-GI-NEXT:    mov v19.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #904]
; CHECK-GI-NEXT:    mov v20.b[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1032]
; CHECK-GI-NEXT:    mov v21.b[6], w9
; CHECK-GI-NEXT:    mov v16.b[7], w11
; CHECK-GI-NEXT:    mov v17.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1160]
; CHECK-GI-NEXT:    ldr w9, [sp, #1288]
; CHECK-GI-NEXT:    mov v18.b[7], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1416]
; CHECK-GI-NEXT:    ldr w11, [sp, #784]
; CHECK-GI-NEXT:    mov v19.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #912]
; CHECK-GI-NEXT:    mov v20.b[7], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1040]
; CHECK-GI-NEXT:    mov v21.b[7], w10
; CHECK-GI-NEXT:    mov v16.b[8], w11
; CHECK-GI-NEXT:    mov v17.b[8], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1168]
; CHECK-GI-NEXT:    ldr w10, [sp, #1296]
; CHECK-GI-NEXT:    mov v18.b[8], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1424]
; CHECK-GI-NEXT:    ldr w11, [sp, #792]
; CHECK-GI-NEXT:    mov v19.b[8], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #920]
; CHECK-GI-NEXT:    mov v20.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1048]
; CHECK-GI-NEXT:    mov v21.b[8], w9
; CHECK-GI-NEXT:    mov v16.b[9], w11
; CHECK-GI-NEXT:    mov v17.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1176]
; CHECK-GI-NEXT:    ldr w9, [sp, #1304]
; CHECK-GI-NEXT:    mov v18.b[9], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1432]
; CHECK-GI-NEXT:    ldr w11, [sp, #800]
; CHECK-GI-NEXT:    mov v19.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #928]
; CHECK-GI-NEXT:    mov v20.b[9], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1056]
; CHECK-GI-NEXT:    mov v21.b[9], w10
; CHECK-GI-NEXT:    mov v16.b[10], w11
; CHECK-GI-NEXT:    mov v17.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1184]
; CHECK-GI-NEXT:    ldr w10, [sp, #1312]
; CHECK-GI-NEXT:    mov v18.b[10], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1440]
; CHECK-GI-NEXT:    ldr w11, [sp, #808]
; CHECK-GI-NEXT:    mov v19.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #936]
; CHECK-GI-NEXT:    mov v20.b[10], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1064]
; CHECK-GI-NEXT:    mov v21.b[10], w9
; CHECK-GI-NEXT:    mov v16.b[11], w11
; CHECK-GI-NEXT:    mov v17.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1192]
; CHECK-GI-NEXT:    ldr w9, [sp, #1320]
; CHECK-GI-NEXT:    mov v18.b[11], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1448]
; CHECK-GI-NEXT:    ldr w11, [sp, #816]
; CHECK-GI-NEXT:    mov v19.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #944]
; CHECK-GI-NEXT:    mov v20.b[11], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1072]
; CHECK-GI-NEXT:    mov v21.b[11], w10
; CHECK-GI-NEXT:    mov v16.b[12], w11
; CHECK-GI-NEXT:    mov v17.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1200]
; CHECK-GI-NEXT:    ldr w10, [sp, #1328]
; CHECK-GI-NEXT:    mov v18.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1456]
; CHECK-GI-NEXT:    ldr w11, [sp, #824]
; CHECK-GI-NEXT:    mov v19.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #952]
; CHECK-GI-NEXT:    mov v20.b[12], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1080]
; CHECK-GI-NEXT:    mov v21.b[12], w9
; CHECK-GI-NEXT:    mov v16.b[13], w11
; CHECK-GI-NEXT:    mov v17.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1208]
; CHECK-GI-NEXT:    ldr w9, [sp, #1336]
; CHECK-GI-NEXT:    mov v18.b[13], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #1464]
; CHECK-GI-NEXT:    ldr w11, [sp, #832]
; CHECK-GI-NEXT:    mov v19.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #960]
; CHECK-GI-NEXT:    mov v20.b[13], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1088]
; CHECK-GI-NEXT:    mov v21.b[13], w10
; CHECK-GI-NEXT:    mov v16.b[14], w11
; CHECK-GI-NEXT:    mov v17.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1216]
; CHECK-GI-NEXT:    ldr w10, [sp, #1344]
; CHECK-GI-NEXT:    mov v18.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #1472]
; CHECK-GI-NEXT:    ldr w11, [sp, #840]
; CHECK-GI-NEXT:    mov v19.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #968]
; CHECK-GI-NEXT:    mov v20.b[14], w10
; CHECK-GI-NEXT:    mov v21.b[14], w9
; CHECK-GI-NEXT:    ldr w10, [sp, #1096]
; CHECK-GI-NEXT:    mov v16.b[15], w11
; CHECK-GI-NEXT:    ldr w9, [sp, #1224]
; CHECK-GI-NEXT:    ldr w11, [sp, #1352]
; CHECK-GI-NEXT:    mov v17.b[15], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #1480]
; CHECK-GI-NEXT:    mov v18.b[15], w10
; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v6.16b
; CHECK-GI-NEXT:    mov v19.b[15], w9
; CHECK-GI-NEXT:    mov v20.b[15], w11
; CHECK-GI-NEXT:    sdot v24.4s, v4.16b, v7.16b
; CHECK-GI-NEXT:    mov v21.b[15], w8
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v2.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    sdot v25.4s, v16.16b, v19.16b
; CHECK-GI-NEXT:    sdot v27.4s, v17.16b, v20.16b
; CHECK-GI-NEXT:    sdot v26.4s, v18.16b, v21.16b
; CHECK-GI-NEXT:    add v1.4s, v24.4s, v1.4s
; CHECK-GI-NEXT:    add v3.4s, v25.4s, v27.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v26.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <48 x i8> %a to <48 x i32>
  %bz = sext <48 x i8> %b to <48 x i32>
  %m1 = mul nuw nsw <48 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
  %cz = sext <48 x i8> %c to <48 x i32>
  %dz = sext <48 x i8> %d to <48 x i32>
  %m2 = mul nuw nsw <48 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w29, -16
; CHECK-SD-NEXT:    add x8, sp, #272
; CHECK-SD-NEXT:    ldr b4, [sp, #208]
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    ldr b5, [x8]
; CHECK-SD-NEXT:    add x8, sp, #216
; CHECK-SD-NEXT:    add x9, sp, #280
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #224
; CHECK-SD-NEXT:    add x12, sp, #256
; CHECK-SD-NEXT:    mov v0.b[1], w1
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #288
; CHECK-SD-NEXT:    ldr b6, [sp, #976]
; CHECK-SD-NEXT:    add x13, sp, #984
; CHECK-SD-NEXT:    add x10, sp, #264
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #232
; CHECK-SD-NEXT:    ldr b7, [sp, #720]
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #296
; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]
; CHECK-SD-NEXT:    mov v0.b[2], w2
; CHECK-SD-NEXT:    add x13, sp, #784
; CHECK-SD-NEXT:    add x11, sp, #328
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #240
; CHECK-SD-NEXT:    ldr b17, [x13]
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #304
; CHECK-SD-NEXT:    add x13, sp, #1064
; CHECK-SD-NEXT:    add x14, sp, #1080
; CHECK-SD-NEXT:    movi v1.16b, #1
; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
; CHECK-SD-NEXT:    add x8, sp, #248
; CHECK-SD-NEXT:    mov v0.b[3], w3
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]
; CHECK-SD-NEXT:    add x9, sp, #312
; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #320
; CHECK-SD-NEXT:    mov v0.b[4], w4
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
; CHECK-SD-NEXT:    add x9, sp, #16
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]
; CHECK-SD-NEXT:    add x12, sp, #1040
; CHECK-SD-NEXT:    ldr b16, [x12]
; CHECK-SD-NEXT:    add x12, sp, #1048
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
; CHECK-SD-NEXT:    mov v0.b[5], w5
; CHECK-SD-NEXT:    add x8, sp, #24
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x12]
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]
; CHECK-SD-NEXT:    add x10, sp, #992
; CHECK-SD-NEXT:    add x12, sp, #1056
; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x10]
; CHECK-SD-NEXT:    add x10, sp, #728
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x11]
; CHECK-SD-NEXT:    add x11, sp, #1000
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x12]
; CHECK-SD-NEXT:    add x12, sp, #792
; CHECK-SD-NEXT:    mov v0.b[6], w6
; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x12]
; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]
; CHECK-SD-NEXT:    add x12, sp, #736
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x12]
; CHECK-SD-NEXT:    add x10, sp, #1008
; CHECK-SD-NEXT:    add x11, sp, #1072
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]
; CHECK-SD-NEXT:    add x13, sp, #800
; CHECK-SD-NEXT:    zip1 v5.2d, v4.2d, v5.2d
; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x13]
; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]
; CHECK-SD-NEXT:    add x13, sp, #808
; CHECK-SD-NEXT:    mov v0.b[7], w7
; CHECK-SD-NEXT:    add x10, sp, #1016
; CHECK-SD-NEXT:    add x12, sp, #32
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
; CHECK-SD-NEXT:    add x11, sp, #744
; CHECK-SD-NEXT:    ldr b4, [sp, #80]
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x11]
; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x13]
; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
; CHECK-SD-NEXT:    add x10, sp, #752
; CHECK-SD-NEXT:    add x11, sp, #816
; CHECK-SD-NEXT:    add x13, sp, #1088
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x14]
; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x9]
; CHECK-SD-NEXT:    add x9, sp, #1024
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x11]
; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #760
; CHECK-SD-NEXT:    add x10, sp, #824
; CHECK-SD-NEXT:    add x11, sp, #1096
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x13]
; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x8]
; CHECK-SD-NEXT:    add x8, sp, #1032
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]
; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]
; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
; CHECK-SD-NEXT:    add x9, sp, #768
; CHECK-SD-NEXT:    add x10, sp, #832
; CHECK-SD-NEXT:    add x8, sp, #40
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]
; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x12]
; CHECK-SD-NEXT:    add x12, sp, #144
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x10]
; CHECK-SD-NEXT:    add x9, sp, #776
; CHECK-SD-NEXT:    add x10, sp, #840
; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v1.16b
; CHECK-SD-NEXT:    ldr b5, [x12]
; CHECK-SD-NEXT:    zip1 v6.2d, v6.2d, v16.2d
; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x8]
; CHECK-SD-NEXT:    add x8, sp, #88
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]
; CHECK-SD-NEXT:    add x10, sp, #912
; CHECK-SD-NEXT:    ldr b16, [x10]
; CHECK-SD-NEXT:    add x9, sp, #152
; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #856
; CHECK-SD-NEXT:    add x10, sp, #920
; CHECK-SD-NEXT:    add x11, sp, #48
; CHECK-SD-NEXT:    sdot v2.4s, v6.16b, v1.16b
; CHECK-SD-NEXT:    zip1 v6.2d, v7.2d, v17.2d
; CHECK-SD-NEXT:    ldr b7, [sp, #848]
; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x11]
; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
; CHECK-SD-NEXT:    add x9, sp, #96
; CHECK-SD-NEXT:    add x10, sp, #160
; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #864
; CHECK-SD-NEXT:    add x11, sp, #928
; CHECK-SD-NEXT:    add x8, sp, #56
; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x10]
; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x8]
; CHECK-SD-NEXT:    add x8, sp, #104
; CHECK-SD-NEXT:    add x9, sp, #168
; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #872
; CHECK-SD-NEXT:    add x10, sp, #936
; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
; CHECK-SD-NEXT:    add x9, sp, #112
; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x10]
; CHECK-SD-NEXT:    add x10, sp, #176
; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]
; CHECK-SD-NEXT:    add x9, sp, #880
; CHECK-SD-NEXT:    add x11, sp, #944
; CHECK-SD-NEXT:    add x8, sp, #64
; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]
; CHECK-SD-NEXT:    add x10, sp, #952
; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x8]
; CHECK-SD-NEXT:    add x8, sp, #120
; CHECK-SD-NEXT:    add x9, sp, #184
; CHECK-SD-NEXT:    add x11, sp, #960
; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
; CHECK-SD-NEXT:    add x8, sp, #888
; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]
; CHECK-SD-NEXT:    add x9, sp, #128
; CHECK-SD-NEXT:    add x10, sp, #192
; CHECK-SD-NEXT:    add x8, sp, #72
; CHECK-SD-NEXT:    sdot v2.4s, v6.16b, v1.16b
; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
; CHECK-SD-NEXT:    add x9, sp, #896
; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]
; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x9]
; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x11]
; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x8]
; CHECK-SD-NEXT:    add x8, sp, #136
; CHECK-SD-NEXT:    add x9, sp, #200
; CHECK-SD-NEXT:    add x10, sp, #968
; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
; CHECK-SD-NEXT:    add x8, sp, #904
; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    zip1 v0.2d, v4.2d, v5.2d
; CHECK-SD-NEXT:    zip1 v4.2d, v7.2d, v16.2d
; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
; CHECK-SD-NEXT:    sdot v2.4s, v4.16b, v1.16b
; CHECK-SD-NEXT:    add v0.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    ldr w9, [sp, #208]
; CHECK-GI-NEXT:    ldr w11, [sp, #80]
; CHECK-GI-NEXT:    fmov s2, w0
; CHECK-GI-NEXT:    ldr w10, [sp, #216]
; CHECK-GI-NEXT:    ldr w12, [sp, #848]
; CHECK-GI-NEXT:    movi d1, #0000000000000000
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    fmov s3, w11
; CHECK-GI-NEXT:    ldr w11, [sp, #720]
; CHECK-GI-NEXT:    ldr w8, [sp, #88]
; CHECK-GI-NEXT:    fmov s6, w12
; CHECK-GI-NEXT:    ldr w9, [sp, #856]
; CHECK-GI-NEXT:    fmov s5, w11
; CHECK-GI-NEXT:    ldr w11, [sp, #984]
; CHECK-GI-NEXT:    mov v2.b[1], w1
; CHECK-GI-NEXT:    mov v4.b[1], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #976]
; CHECK-GI-NEXT:    mov v3.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #728]
; CHECK-GI-NEXT:    mov v6.b[1], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #736]
; CHECK-GI-NEXT:    fmov s7, w10
; CHECK-GI-NEXT:    ldr w10, [sp, #96]
; CHECK-GI-NEXT:    ldr w12, [sp, #16]
; CHECK-GI-NEXT:    mov v5.b[1], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #224]
; CHECK-GI-NEXT:    mov v2.b[2], w2
; CHECK-GI-NEXT:    mov v3.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #864]
; CHECK-GI-NEXT:    movi d0, #0000000000000000
; CHECK-GI-NEXT:    mov v7.b[1], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #992]
; CHECK-GI-NEXT:    mov v4.b[2], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #104]
; CHECK-GI-NEXT:    mov v6.b[2], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #872]
; CHECK-GI-NEXT:    mov v5.b[2], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #232]
; CHECK-GI-NEXT:    mov v2.b[3], w3
; CHECK-GI-NEXT:    mov v3.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #744]
; CHECK-GI-NEXT:    mov v1.s[1], wzr
; CHECK-GI-NEXT:    mov v7.b[2], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1000]
; CHECK-GI-NEXT:    mov v4.b[3], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #112]
; CHECK-GI-NEXT:    mov v6.b[3], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #880]
; CHECK-GI-NEXT:    mov v5.b[3], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #240]
; CHECK-GI-NEXT:    mov v2.b[4], w4
; CHECK-GI-NEXT:    mov v3.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #752]
; CHECK-GI-NEXT:    mov v0.s[1], wzr
; CHECK-GI-NEXT:    mov v7.b[3], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1008]
; CHECK-GI-NEXT:    mov v4.b[4], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #120]
; CHECK-GI-NEXT:    mov v6.b[4], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #888]
; CHECK-GI-NEXT:    mov v5.b[4], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #248]
; CHECK-GI-NEXT:    mov v2.b[5], w5
; CHECK-GI-NEXT:    mov v3.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #760]
; CHECK-GI-NEXT:    movi v16.16b, #1
; CHECK-GI-NEXT:    mov v7.b[4], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
; CHECK-GI-NEXT:    mov v4.b[5], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #128]
; CHECK-GI-NEXT:    mov v6.b[5], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #896]
; CHECK-GI-NEXT:    mov v5.b[5], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #256]
; CHECK-GI-NEXT:    mov v2.b[6], w6
; CHECK-GI-NEXT:    mov v3.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #768]
; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v7.b[5], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1024]
; CHECK-GI-NEXT:    mov v4.b[6], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #136]
; CHECK-GI-NEXT:    mov v6.b[6], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #904]
; CHECK-GI-NEXT:    mov v5.b[6], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #264]
; CHECK-GI-NEXT:    mov v2.b[7], w7
; CHECK-GI-NEXT:    mov v3.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #776]
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v7.b[6], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1032]
; CHECK-GI-NEXT:    mov v4.b[7], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #144]
; CHECK-GI-NEXT:    mov v6.b[7], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #912]
; CHECK-GI-NEXT:    mov v5.b[7], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #272]
; CHECK-GI-NEXT:    mov v2.b[8], w12
; CHECK-GI-NEXT:    mov v3.b[8], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #784]
; CHECK-GI-NEXT:    ldr w12, [sp, #24]
; CHECK-GI-NEXT:    mov v7.b[7], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1040]
; CHECK-GI-NEXT:    mov v4.b[8], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #152]
; CHECK-GI-NEXT:    mov v6.b[8], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #920]
; CHECK-GI-NEXT:    mov v5.b[8], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #280]
; CHECK-GI-NEXT:    mov v2.b[9], w12
; CHECK-GI-NEXT:    mov v3.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #792]
; CHECK-GI-NEXT:    ldr w12, [sp, #32]
; CHECK-GI-NEXT:    mov v7.b[8], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1048]
; CHECK-GI-NEXT:    mov v4.b[9], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #160]
; CHECK-GI-NEXT:    mov v6.b[9], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #928]
; CHECK-GI-NEXT:    mov v5.b[9], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #288]
; CHECK-GI-NEXT:    mov v2.b[10], w12
; CHECK-GI-NEXT:    mov v3.b[10], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #800]
; CHECK-GI-NEXT:    ldr w12, [sp, #40]
; CHECK-GI-NEXT:    mov v7.b[9], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1056]
; CHECK-GI-NEXT:    mov v4.b[10], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #168]
; CHECK-GI-NEXT:    mov v6.b[10], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #936]
; CHECK-GI-NEXT:    mov v5.b[10], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #296]
; CHECK-GI-NEXT:    mov v2.b[11], w12
; CHECK-GI-NEXT:    mov v3.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #808]
; CHECK-GI-NEXT:    ldr w12, [sp, #48]
; CHECK-GI-NEXT:    mov v7.b[10], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1064]
; CHECK-GI-NEXT:    mov v4.b[11], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #176]
; CHECK-GI-NEXT:    mov v6.b[11], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #944]
; CHECK-GI-NEXT:    mov v5.b[11], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #304]
; CHECK-GI-NEXT:    mov v2.b[12], w12
; CHECK-GI-NEXT:    mov v3.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #816]
; CHECK-GI-NEXT:    ldr w12, [sp, #56]
; CHECK-GI-NEXT:    mov v7.b[11], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1072]
; CHECK-GI-NEXT:    mov v4.b[12], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #184]
; CHECK-GI-NEXT:    mov v6.b[12], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #952]
; CHECK-GI-NEXT:    mov v5.b[12], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #312]
; CHECK-GI-NEXT:    mov v2.b[13], w12
; CHECK-GI-NEXT:    mov v3.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #824]
; CHECK-GI-NEXT:    ldr w12, [sp, #64]
; CHECK-GI-NEXT:    mov v7.b[12], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1080]
; CHECK-GI-NEXT:    mov v4.b[13], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #192]
; CHECK-GI-NEXT:    mov v6.b[13], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #960]
; CHECK-GI-NEXT:    mov v5.b[13], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #320]
; CHECK-GI-NEXT:    mov v2.b[14], w12
; CHECK-GI-NEXT:    mov v3.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #832]
; CHECK-GI-NEXT:    ldr w12, [sp, #72]
; CHECK-GI-NEXT:    mov v7.b[13], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1088]
; CHECK-GI-NEXT:    mov v4.b[14], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #200]
; CHECK-GI-NEXT:    mov v6.b[14], w10
; CHECK-GI-NEXT:    ldr w10, [sp, #968]
; CHECK-GI-NEXT:    mov v5.b[14], w9
; CHECK-GI-NEXT:    ldr w9, [sp, #328]
; CHECK-GI-NEXT:    mov v2.b[15], w12
; CHECK-GI-NEXT:    mov v3.b[15], w8
; CHECK-GI-NEXT:    ldr w8, [sp, #840]
; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v7.b[14], w11
; CHECK-GI-NEXT:    ldr w11, [sp, #1096]
; CHECK-GI-NEXT:    mov v4.b[15], w9
; CHECK-GI-NEXT:    mov v6.b[15], w10
; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v5.b[15], w8
; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v1.s[2], wzr
; CHECK-GI-NEXT:    mov v0.s[2], wzr
; CHECK-GI-NEXT:    sdot v17.4s, v2.16b, v16.16b
; CHECK-GI-NEXT:    sdot v18.4s, v3.16b, v16.16b
; CHECK-GI-NEXT:    mov v7.b[15], w11
; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v16.16b
; CHECK-GI-NEXT:    sdot v21.4s, v6.16b, v16.16b
; CHECK-GI-NEXT:    sdot v20.4s, v5.16b, v16.16b
; CHECK-GI-NEXT:    mov v1.s[3], wzr
; CHECK-GI-NEXT:    mov v0.s[3], wzr
; CHECK-GI-NEXT:    add v2.4s, v17.4s, v18.4s
; CHECK-GI-NEXT:    sdot v22.4s, v7.16b, v16.16b
; CHECK-GI-NEXT:    add v1.4s, v19.4s, v1.4s
; CHECK-GI-NEXT:    add v3.4s, v20.4s, v21.4s
; CHECK-GI-NEXT:    add v0.4s, v22.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <48 x i8> %a to <48 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
  %cz = sext <48 x i8> %c to <48 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_udot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v64i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]
; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]
; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b
; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b
; CHECK-SD-NEXT:    ldp q2, q3, [x0]
; CHECK-SD-NEXT:    ldp q4, q5, [x1]
; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b
; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v64i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q1, q2, [x0]
; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]
; CHECK-GI-NEXT:    ldp q16, q17, [x1]
; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]
; CHECK-GI-NEXT:    udot v0.4s, v16.16b, v1.16b
; CHECK-GI-NEXT:    udot v4.4s, v17.16b, v2.16b
; CHECK-GI-NEXT:    udot v5.4s, v18.16b, v6.16b
; CHECK-GI-NEXT:    udot v3.4s, v19.16b, v7.16b
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <64 x i8>, ptr %a
  %1 = zext <64 x i8> %0 to <64 x i32>
  %2 = load <64 x i8>, ptr %b
  %3 = zext <64 x i8> %2 to <64 x i32>
  %4 = mul nuw nsw <64 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
  %op.extra = add i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_udot_v64i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v64i8_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.16b, #1
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q3, q4, [x0, #32]
; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b
; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
; CHECK-SD-NEXT:    ldp q3, q4, [x0]
; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b
; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
; CHECK-SD-NEXT:    add v0.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_udot_v64i8_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.16b, #1
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q5, q6, [x0]
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q7, q16, [x0, #32]
; CHECK-GI-NEXT:    udot v1.4s, v5.16b, v0.16b
; CHECK-GI-NEXT:    udot v3.4s, v6.16b, v0.16b
; CHECK-GI-NEXT:    udot v2.4s, v16.16b, v0.16b
; CHECK-GI-NEXT:    udot v4.4s, v7.16b, v0.16b
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    add v1.4s, v4.4s, v2.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <64 x i8>, ptr %a1
  %1 = zext <64 x i8> %0 to <64 x i32>
  %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)
  ret i32 %2
}
define i32 @test_sdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_sdot_v64i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]
; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]
; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b
; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b
; CHECK-SD-NEXT:    ldp q2, q3, [x0]
; CHECK-SD-NEXT:    ldp q4, q5, [x1]
; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b
; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v64i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q1, q2, [x0]
; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]
; CHECK-GI-NEXT:    ldp q16, q17, [x1]
; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]
; CHECK-GI-NEXT:    sdot v0.4s, v16.16b, v1.16b
; CHECK-GI-NEXT:    sdot v4.4s, v17.16b, v2.16b
; CHECK-GI-NEXT:    sdot v5.4s, v18.16b, v6.16b
; CHECK-GI-NEXT:    sdot v3.4s, v19.16b, v7.16b
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <64 x i8>, ptr %a
  %1 = sext <64 x i8> %0 to <64 x i32>
  %2 = load <64 x i8>, ptr %b
  %3 = sext <64 x i8> %2 to <64 x i32>
  %4 = mul nsw <64 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_sdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v64i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q20, q21, [sp, #96]
; CHECK-SD-NEXT:    ldp q22, q23, [sp, #32]
; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v7.16b
; CHECK-SD-NEXT:    sdot v18.4s, v2.16b, v6.16b
; CHECK-SD-NEXT:    sdot v19.4s, v23.16b, v21.16b
; CHECK-SD-NEXT:    sdot v17.4s, v22.16b, v20.16b
; CHECK-SD-NEXT:    ldp q2, q3, [sp, #64]
; CHECK-SD-NEXT:    ldp q6, q7, [sp]
; CHECK-SD-NEXT:    sdot v16.4s, v1.16b, v5.16b
; CHECK-SD-NEXT:    sdot v18.4s, v0.16b, v4.16b
; CHECK-SD-NEXT:    sdot v19.4s, v7.16b, v3.16b
; CHECK-SD-NEXT:    sdot v17.4s, v6.16b, v2.16b
; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s
; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v64i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q16, q17, [sp]
; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v27.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q19, q20, [sp, #32]
; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v4.16b
; CHECK-GI-NEXT:    ldp q0, q4, [sp, #64]
; CHECK-GI-NEXT:    sdot v21.4s, v1.16b, v5.16b
; CHECK-GI-NEXT:    ldp q1, q5, [sp, #96]
; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v6.16b
; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v7.16b
; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v0.16b
; CHECK-GI-NEXT:    sdot v26.4s, v17.16b, v4.16b
; CHECK-GI-NEXT:    sdot v27.4s, v19.16b, v1.16b
; CHECK-GI-NEXT:    sdot v25.4s, v20.16b, v5.16b
; CHECK-GI-NEXT:    add v0.4s, v18.4s, v21.4s
; CHECK-GI-NEXT:    add v1.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    add v2.4s, v24.4s, v26.4s
; CHECK-GI-NEXT:    add v3.4s, v27.4s, v25.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <64 x i8> %a to <64 x i32>
  %bz = sext <64 x i8> %b to <64 x i32>
  %m1 = mul nuw nsw <64 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)
  %cz = sext <64 x i8> %c to <64 x i32>
  %dz = sext <64 x i8> %d to <64 x i32>
  %m2 = mul nuw nsw <64 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_sdot_v64i8_double_nomla(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
; CHECK-SD-LABEL: test_sdot_v64i8_double_nomla:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.16b, #1
; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q17, q18, [sp, #32]
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v4.16b
; CHECK-SD-NEXT:    sdot v6.4s, v17.16b, v4.16b
; CHECK-SD-NEXT:    sdot v7.4s, v2.16b, v4.16b
; CHECK-SD-NEXT:    ldp q2, q3, [sp]
; CHECK-SD-NEXT:    sdot v16.4s, v18.16b, v4.16b
; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b
; CHECK-SD-NEXT:    sdot v6.4s, v2.16b, v4.16b
; CHECK-SD-NEXT:    sdot v7.4s, v0.16b, v4.16b
; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v4.16b
; CHECK-SD-NEXT:    add v0.4s, v7.4s, v5.4s
; CHECK-SD-NEXT:    add v1.4s, v6.4s, v16.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_sdot_v64i8_double_nomla:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.16b, #1
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v6.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
; CHECK-GI-NEXT:    ldp q21, q22, [sp]
; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v4.16b
; CHECK-GI-NEXT:    sdot v6.4s, v1.16b, v4.16b
; CHECK-GI-NEXT:    ldp q0, q1, [sp, #32]
; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b
; CHECK-GI-NEXT:    sdot v16.4s, v3.16b, v4.16b
; CHECK-GI-NEXT:    sdot v17.4s, v21.16b, v4.16b
; CHECK-GI-NEXT:    sdot v19.4s, v22.16b, v4.16b
; CHECK-GI-NEXT:    sdot v20.4s, v0.16b, v4.16b
; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v4.16b
; CHECK-GI-NEXT:    add v0.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    add v1.4s, v7.4s, v16.4s
; CHECK-GI-NEXT:    add v2.4s, v17.4s, v19.4s
; CHECK-GI-NEXT:    add v3.4s, v20.4s, v18.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    fmov w9, s1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %az = sext <64 x i8> %a to <64 x i32>
  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az)
  %cz = sext <64 x i8> %c to <64 x i32>
  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz)
  %x = add i32 %r1, %r2
  ret i32 %x
}

define i32 @test_usdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_usdot_v64i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]
; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]
; CHECK-SD-NEXT:    usdot v1.4s, v3.16b, v5.16b
; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v4.16b
; CHECK-SD-NEXT:    ldp q2, q3, [x0]
; CHECK-SD-NEXT:    ldp q4, q5, [x1]
; CHECK-SD-NEXT:    usdot v1.4s, v3.16b, v5.16b
; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v4.16b
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w8, s0
; CHECK-SD-NEXT:    add w0, w8, w2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v64i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldp q0, q1, [x0]
; CHECK-GI-NEXT:    ldp q2, q5, [x1]
; CHECK-GI-NEXT:    ldp q3, q4, [x0, #32]
; CHECK-GI-NEXT:    ldp q6, q7, [x1, #32]
; CHECK-GI-NEXT:    ushll v20.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v16.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    sshll v17.8h, v5.8b, #0
; CHECK-GI-NEXT:    sshll2 v5.8h, v5.16b, #0
; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
; CHECK-GI-NEXT:    ushll v21.8h, v1.8b, #0
; CHECK-GI-NEXT:    sshll v18.8h, v6.8b, #0
; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #0
; CHECK-GI-NEXT:    sshll v19.8h, v7.8b, #0
; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #0
; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT:    ushll v22.8h, v3.8b, #0
; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    ushll v23.8h, v4.8b, #0
; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #0
; CHECK-GI-NEXT:    smull v24.4s, v16.4h, v20.4h
; CHECK-GI-NEXT:    smull v25.4s, v2.4h, v0.4h
; CHECK-GI-NEXT:    smull v26.4s, v17.4h, v21.4h
; CHECK-GI-NEXT:    smull v27.4s, v5.4h, v1.4h
; CHECK-GI-NEXT:    smull v28.4s, v18.4h, v22.4h
; CHECK-GI-NEXT:    smull v29.4s, v6.4h, v3.4h
; CHECK-GI-NEXT:    smull v30.4s, v19.4h, v23.4h
; CHECK-GI-NEXT:    smull v31.4s, v7.4h, v4.4h
; CHECK-GI-NEXT:    smlal2 v24.4s, v16.8h, v20.8h
; CHECK-GI-NEXT:    smlal2 v25.4s, v2.8h, v0.8h
; CHECK-GI-NEXT:    smlal2 v26.4s, v17.8h, v21.8h
; CHECK-GI-NEXT:    smlal2 v27.4s, v5.8h, v1.8h
; CHECK-GI-NEXT:    smlal2 v28.4s, v18.8h, v22.8h
; CHECK-GI-NEXT:    smlal2 v29.4s, v6.8h, v3.8h
; CHECK-GI-NEXT:    smlal2 v30.4s, v19.8h, v23.8h
; CHECK-GI-NEXT:    smlal2 v31.4s, v7.8h, v4.8h
; CHECK-GI-NEXT:    add v0.4s, v24.4s, v25.4s
; CHECK-GI-NEXT:    add v1.4s, v26.4s, v27.4s
; CHECK-GI-NEXT:    add v2.4s, v28.4s, v29.4s
; CHECK-GI-NEXT:    add v3.4s, v30.4s, v31.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s0
; CHECK-GI-NEXT:    add w0, w8, w2
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <64 x i8>, ptr %a
  %1 = zext <64 x i8> %0 to <64 x i32>
  %2 = load <64 x i8>, ptr %b
  %3 = sext <64 x i8> %2 to <64 x i32>
  %4 = mul nsw <64 x i32> %3, %1
  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
  %op.extra = add nsw i32 %5, %sum
  ret i32 %op.extra
}

define i32 @test_usdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
; CHECK-SD-LABEL: test_usdot_v64i8_double:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
; CHECK-SD-NEXT:    ldp q20, q21, [sp, #96]
; CHECK-SD-NEXT:    ldp q22, q23, [sp, #32]
; CHECK-SD-NEXT:    usdot v16.4s, v3.16b, v7.16b
; CHECK-SD-NEXT:    usdot v18.4s, v2.16b, v6.16b
; CHECK-SD-NEXT:    usdot v19.4s, v23.16b, v21.16b
; CHECK-SD-NEXT:    usdot v17.4s, v22.16b, v20.16b
; CHECK-SD-NEXT:    ldp q2, q3, [sp, #64]
; CHECK-SD-NEXT:    ldp q6, q7, [sp]
; CHECK-SD-NEXT:    usdot v16.4s, v1.16b, v5.16b
; CHECK-SD-NEXT:    usdot v18.4s, v0.16b, v4.16b
; CHECK-SD-NEXT:    usdot v19.4s, v7.16b, v3.16b
; CHECK-SD-NEXT:    usdot v17.4s, v6.16b, v2.16b
; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s
; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    addv s0, v0.4s
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: test_usdot_v64i8_double:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sub sp, sp, #240
; CHECK-GI-NEXT:    stp d15, d14, [sp, #160] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d13, d12, [sp, #176] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d11, d10, [sp, #192] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp d9, d8, [sp, #208] // 16-byte Folded Spill
; CHECK-GI-NEXT:    str x29, [sp, #224] // 8-byte Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 240
; CHECK-GI-NEXT:    .cfi_offset w29, -16
; CHECK-GI-NEXT:    .cfi_offset b8, -24
; CHECK-GI-NEXT:    .cfi_offset b9, -32
; CHECK-GI-NEXT:    .cfi_offset b10, -40
; CHECK-GI-NEXT:    .cfi_offset b11, -48
; CHECK-GI-NEXT:    .cfi_offset b12, -56
; CHECK-GI-NEXT:    .cfi_offset b13, -64
; CHECK-GI-NEXT:    .cfi_offset b14, -72
; CHECK-GI-NEXT:    .cfi_offset b15, -80
; CHECK-GI-NEXT:    ushll v31.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll2 v8.8h, v0.16b, #0
; CHECK-GI-NEXT:    ldr x29, [sp, #224] // 8-byte Reload
; CHECK-GI-NEXT:    sshll v11.8h, v4.8b, #0
; CHECK-GI-NEXT:    sshll2 v12.8h, v4.16b, #0
; CHECK-GI-NEXT:    ushll v9.8h, v1.8b, #0
; CHECK-GI-NEXT:    ushll2 v10.8h, v1.16b, #0
; CHECK-GI-NEXT:    ldp q25, q22, [sp, #240]
; CHECK-GI-NEXT:    sshll v13.8h, v5.8b, #0
; CHECK-GI-NEXT:    sshll2 v14.8h, v5.16b, #0
; CHECK-GI-NEXT:    ushll v0.8h, v2.8b, #0
; CHECK-GI-NEXT:    smull v19.4s, v31.4h, v11.4h
; CHECK-GI-NEXT:    ldp q21, q18, [sp, #272]
; CHECK-GI-NEXT:    smull v20.4s, v8.4h, v12.4h
; CHECK-GI-NEXT:    sshll v4.8h, v6.8b, #0
; CHECK-GI-NEXT:    sshll2 v5.8h, v6.16b, #0
; CHECK-GI-NEXT:    smull v28.4s, v9.4h, v13.4h
; CHECK-GI-NEXT:    ldp q17, q16, [sp, #304]
; CHECK-GI-NEXT:    smull v27.4s, v10.4h, v14.4h
; CHECK-GI-NEXT:    sshll v6.8h, v7.8b, #0
; CHECK-GI-NEXT:    ushll2 v1.8h, v2.16b, #0
; CHECK-GI-NEXT:    smlal2 v19.4s, v31.8h, v11.8h
; CHECK-GI-NEXT:    ldp q30, q29, [sp, #336]
; CHECK-GI-NEXT:    smlal2 v20.4s, v8.8h, v12.8h
; CHECK-GI-NEXT:    ushll v2.8h, v3.8b, #0
; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #0
; CHECK-GI-NEXT:    smlal2 v28.4s, v9.8h, v13.8h
; CHECK-GI-NEXT:    stp q0, q6, [sp, #48] // 32-byte Folded Spill
; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #0
; CHECK-GI-NEXT:    smlal2 v27.4s, v10.8h, v14.8h
; CHECK-GI-NEXT:    smull v26.4s, v0.4h, v4.4h
; CHECK-GI-NEXT:    ushll v31.8h, v25.8b, #0
; CHECK-GI-NEXT:    str q19, [sp, #144] // 16-byte Spill
; CHECK-GI-NEXT:    ushll2 v25.8h, v25.16b, #0
; CHECK-GI-NEXT:    ushll v8.8h, v22.8b, #0
; CHECK-GI-NEXT:    stp q2, q1, [sp] // 32-byte Folded Spill
; CHECK-GI-NEXT:    ushll2 v22.8h, v22.16b, #0
; CHECK-GI-NEXT:    ushll v9.8h, v21.8b, #0
; CHECK-GI-NEXT:    stp q5, q28, [sp, #80] // 32-byte Folded Spill
; CHECK-GI-NEXT:    ushll2 v21.8h, v21.16b, #0
; CHECK-GI-NEXT:    ushll v10.8h, v18.8b, #0
; CHECK-GI-NEXT:    stp q4, q20, [sp, #112] // 32-byte Folded Spill
; CHECK-GI-NEXT:    ushll2 v20.8h, v18.16b, #0
; CHECK-GI-NEXT:    sshll v11.8h, v17.8b, #0
; CHECK-GI-NEXT:    str q27, [sp, #32] // 16-byte Spill
; CHECK-GI-NEXT:    ldr q28, [sp, #112] // 16-byte Reload
; CHECK-GI-NEXT:    ldr q27, [sp, #48] // 16-byte Reload
; CHECK-GI-NEXT:    sshll2 v19.8h, v17.16b, #0
; CHECK-GI-NEXT:    sshll v12.8h, v16.8b, #0
; CHECK-GI-NEXT:    sshll2 v18.8h, v16.16b, #0
; CHECK-GI-NEXT:    sshll v13.8h, v30.8b, #0
; CHECK-GI-NEXT:    sshll2 v30.8h, v30.16b, #0
; CHECK-GI-NEXT:    sshll v14.8h, v29.8b, #0
; CHECK-GI-NEXT:    sshll2 v29.8h, v29.16b, #0
; CHECK-GI-NEXT:    smull v23.4s, v1.4h, v5.4h
; CHECK-GI-NEXT:    smull v15.4s, v3.4h, v7.4h
; CHECK-GI-NEXT:    smull v24.4s, v2.4h, v6.4h
; CHECK-GI-NEXT:    smull v17.4s, v31.4h, v11.4h
; CHECK-GI-NEXT:    smull v6.4s, v25.4h, v19.4h
; CHECK-GI-NEXT:    smull v16.4s, v8.4h, v12.4h
; CHECK-GI-NEXT:    smull v4.4s, v22.4h, v18.4h
; CHECK-GI-NEXT:    smull v5.4s, v9.4h, v13.4h
; CHECK-GI-NEXT:    smull v2.4s, v21.4h, v30.4h
; CHECK-GI-NEXT:    smull v1.4s, v10.4h, v14.4h
; CHECK-GI-NEXT:    smull v0.4s, v20.4h, v29.4h
; CHECK-GI-NEXT:    smlal2 v26.4s, v27.8h, v28.8h
; CHECK-GI-NEXT:    ldr q28, [sp, #80] // 16-byte Reload
; CHECK-GI-NEXT:    ldr q27, [sp, #16] // 16-byte Reload
; CHECK-GI-NEXT:    smlal2 v15.4s, v3.8h, v7.8h
; CHECK-GI-NEXT:    ldp q7, q3, [sp, #128] // 32-byte Folded Reload
; CHECK-GI-NEXT:    smlal2 v23.4s, v27.8h, v28.8h
; CHECK-GI-NEXT:    ldr q28, [sp, #64] // 16-byte Reload
; CHECK-GI-NEXT:    ldr q27, [sp] // 16-byte Reload
; CHECK-GI-NEXT:    smlal2 v17.4s, v31.8h, v11.8h
; CHECK-GI-NEXT:    smlal2 v6.4s, v25.8h, v19.8h
; CHECK-GI-NEXT:    smlal2 v16.4s, v8.8h, v12.8h
; CHECK-GI-NEXT:    smlal2 v24.4s, v27.8h, v28.8h
; CHECK-GI-NEXT:    smlal2 v4.4s, v22.8h, v18.8h
; CHECK-GI-NEXT:    ldr q18, [sp, #32] // 16-byte Reload
; CHECK-GI-NEXT:    smlal2 v5.4s, v9.8h, v13.8h
; CHECK-GI-NEXT:    ldp d9, d8, [sp, #208] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp d13, d12, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT:    smlal2 v2.4s, v21.8h, v30.8h
; CHECK-GI-NEXT:    smlal2 v1.4s, v10.8h, v14.8h
; CHECK-GI-NEXT:    ldp d11, d10, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT:    smlal2 v0.4s, v20.8h, v29.8h
; CHECK-GI-NEXT:    add v3.4s, v3.4s, v7.4s
; CHECK-GI-NEXT:    ldr q7, [sp, #96] // 16-byte Reload
; CHECK-GI-NEXT:    add v19.4s, v24.4s, v15.4s
; CHECK-GI-NEXT:    ldp d15, d14, [sp, #160] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add v7.4s, v7.4s, v18.4s
; CHECK-GI-NEXT:    add v18.4s, v26.4s, v23.4s
; CHECK-GI-NEXT:    add v6.4s, v17.4s, v6.4s
; CHECK-GI-NEXT:    add v4.4s, v16.4s, v4.4s
; CHECK-GI-NEXT:    add v2.4s, v5.4s, v2.4s
; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v3.4s, v7.4s
; CHECK-GI-NEXT:    add v3.4s, v18.4s, v19.4s
; CHECK-GI-NEXT:    add v4.4s, v6.4s, v4.4s
; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    add v0.4s, v4.4s, v0.4s
; CHECK-GI-NEXT:    addv s1, v1.4s
; CHECK-GI-NEXT:    addv s0, v0.4s
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    add sp, sp, #240
; CHECK-GI-NEXT:    ret
entry:
  %az = zext <64 x i8> %a to <64 x i32>
  %bz = sext <64 x i8> %b to <64 x i32>
  %m1 = mul nuw nsw <64 x i32> %az, %bz
  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)
  %cz = zext <64 x i8> %c to <64 x i32>
  %dz = sext <64 x i8> %d to <64 x i32>
  %m2 = mul nuw nsw <64 x i32> %cz, %dz
  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)
  %x = add i32 %r1, %r2
  ret i32 %x
}