; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE
; RUN: llc -enable-subreg-liveness -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE2
; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=SVE2
; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE

; This test only tests the legal types for a given vector width, as mulh nodes
; do not get generated for non-legal types.

target triple = "aarch64-unknown-linux-gnu"

;
; SMULH
;

define <4 x i8> @smulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
; SVE-LABEL: smulh_v4i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    sxtb z0.h, p0/m, z0.h
; SVE-NEXT:    sxtb z1.h, p0/m, z1.h
; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    lsr z0.h, z0.h, #4
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v4i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ptrue p0.h, vl4
; SVE2-NEXT:    sxtb z0.h, p0/m, z0.h
; SVE2-NEXT:    sxtb z1.h, p0/m, z1.h
; SVE2-NEXT:    mul z0.h, z0.h, z1.h
; SVE2-NEXT:    lsr z0.h, z0.h, #4
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v4i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #18]
; NONEON-NOSVE-NEXT:    mul w8, w8, w12
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w9, w9, w13
; NONEON-NOSVE-NEXT:    mul w10, w10, w14
; NONEON-NOSVE-NEXT:    mul w11, w11, w12
; NONEON-NOSVE-NEXT:    ubfx w8, w8, #4, #12
; NONEON-NOSVE-NEXT:    ubfx w9, w9, #4, #12
; NONEON-NOSVE-NEXT:    ubfx w10, w10, #4, #12
; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
; NONEON-NOSVE-NEXT:    ubfx w8, w11, #4, #12
; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <4 x i8> %op1 to <4 x i16>
  %2 = sext <4 x i8> %op2 to <4 x i16>
  %mul = mul <4 x i16> %1, %2
  %shr = lshr <4 x i16> %mul, splat (i16 4)
  %res = trunc <4 x i16> %shr to <4 x i8>
  ret <4 x i8> %res
}

define <8 x i8> @smulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
; SVE-LABEL: smulh_v8i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.b, vl8
; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v8i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.b, z0.b, z1.b
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v8i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #15]
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #23]
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #13]
; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #22]
; NONEON-NOSVE-NEXT:    mul w15, w15, w16
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #21]
; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    mul w14, w14, w17
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #9]
; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #10]
; NONEON-NOSVE-NEXT:    mul w13, w13, w16
; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #11]
; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w12, w12, w18
; NONEON-NOSVE-NEXT:    lsr w15, w15, #8
; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #18]
; NONEON-NOSVE-NEXT:    lsr w14, w14, #8
; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #17]
; NONEON-NOSVE-NEXT:    mul w8, w8, w17
; NONEON-NOSVE-NEXT:    lsr w13, w13, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w0
; NONEON-NOSVE-NEXT:    lsr w12, w12, #8
; NONEON-NOSVE-NEXT:    strb w15, [sp, #31]
; NONEON-NOSVE-NEXT:    mul w10, w10, w16
; NONEON-NOSVE-NEXT:    strb w14, [sp, #30]
; NONEON-NOSVE-NEXT:    mul w9, w9, w18
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    strb w13, [sp, #29]
; NONEON-NOSVE-NEXT:    lsr w11, w11, #8
; NONEON-NOSVE-NEXT:    strb w12, [sp, #28]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    strb w11, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w10, [sp, #26]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <8 x i8> %op1 to <8 x i16>
  %2 = sext <8 x i8> %op2 to <8 x i16>
  %mul = mul <8 x i16> %1, %2
  %shr = lshr <8 x i16> %mul, splat (i16 8)
  %res = trunc <8 x i16> %shr to <8 x i8>
  ret <8 x i8> %res
}

define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
; SVE-LABEL: smulh_v16i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.b, vl16
; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v16i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.b, z0.b, z1.b
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v16i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    str x27, [sp, #80] // 8-byte Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #96] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #112] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #128] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -80
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp]
; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #40]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsb w6, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrsb w7, [sp, #45]
; NONEON-NOSVE-NEXT:    ldrsb w19, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsb w20, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    str d0, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #51]
; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #63]
; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrsb w25, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrsb w26, [sp, #60]
; NONEON-NOSVE-NEXT:    str d1, [sp, #88]
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #52]
; NONEON-NOSVE-NEXT:    mul w20, w20, w21
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #53]
; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #54]
; NONEON-NOSVE-NEXT:    mul w19, w19, w23
; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #55]
; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #40]
; NONEON-NOSVE-NEXT:    mul w7, w7, w25
; NONEON-NOSVE-NEXT:    ldrsb w2, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrsb w3, [sp, #42]
; NONEON-NOSVE-NEXT:    mul w6, w6, w26
; NONEON-NOSVE-NEXT:    lsr w20, w20, #8
; NONEON-NOSVE-NEXT:    ldrsb w4, [sp, #43]
; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #88]
; NONEON-NOSVE-NEXT:    lsr w19, w19, #8
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #89]
; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #90]
; NONEON-NOSVE-NEXT:    lsr w7, w7, #8
; NONEON-NOSVE-NEXT:    ldrsb w1, [sp, #91]
; NONEON-NOSVE-NEXT:    ldrsb w5, [sp, #92]
; NONEON-NOSVE-NEXT:    mul w9, w9, w16
; NONEON-NOSVE-NEXT:    lsr w6, w6, #8
; NONEON-NOSVE-NEXT:    ldrsb w22, [sp, #93]
; NONEON-NOSVE-NEXT:    ldrsb w24, [sp, #94]
; NONEON-NOSVE-NEXT:    mul w11, w11, w1
; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #95]
; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #56]
; NONEON-NOSVE-NEXT:    mul w12, w12, w5
; NONEON-NOSVE-NEXT:    ldrsb w27, [sp, #59]
; NONEON-NOSVE-NEXT:    ldrsb w25, [sp, #58]
; NONEON-NOSVE-NEXT:    mul w15, w15, w24
; NONEON-NOSVE-NEXT:    ldrsb w26, [sp, #57]
; NONEON-NOSVE-NEXT:    mul w0, w0, w23
; NONEON-NOSVE-NEXT:    lsr w11, w11, #8
; NONEON-NOSVE-NEXT:    mul w4, w4, w27
; NONEON-NOSVE-NEXT:    lsr w12, w12, #8
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    mul w3, w3, w25
; NONEON-NOSVE-NEXT:    lsr w15, w15, #8
; NONEON-NOSVE-NEXT:    strb w20, [sp, #79]
; NONEON-NOSVE-NEXT:    mul w2, w2, w26
; NONEON-NOSVE-NEXT:    lsr w0, w0, #8
; NONEON-NOSVE-NEXT:    strb w19, [sp, #78]
; NONEON-NOSVE-NEXT:    mul w17, w17, w21
; NONEON-NOSVE-NEXT:    lsr w4, w4, #8
; NONEON-NOSVE-NEXT:    strb w7, [sp, #77]
; NONEON-NOSVE-NEXT:    mul w13, w13, w22
; NONEON-NOSVE-NEXT:    lsr w3, w3, #8
; NONEON-NOSVE-NEXT:    strb w6, [sp, #76]
; NONEON-NOSVE-NEXT:    mul w10, w10, w18
; NONEON-NOSVE-NEXT:    lsr w2, w2, #8
; NONEON-NOSVE-NEXT:    strb w4, [sp, #75]
; NONEON-NOSVE-NEXT:    mul w8, w8, w14
; NONEON-NOSVE-NEXT:    lsr w17, w17, #8
; NONEON-NOSVE-NEXT:    strb w3, [sp, #74]
; NONEON-NOSVE-NEXT:    lsr w13, w13, #8
; NONEON-NOSVE-NEXT:    strb w2, [sp, #73]
; NONEON-NOSVE-NEXT:    ldr x27, [sp, #80] // 8-byte Reload
; NONEON-NOSVE-NEXT:    lsr w10, w10, #8
; NONEON-NOSVE-NEXT:    strb w0, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    strb w17, [sp, #71]
; NONEON-NOSVE-NEXT:    strb w15, [sp, #70]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w13, [sp, #69]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #128] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w12, [sp, #68]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #112] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w11, [sp, #67]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #96] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w10, [sp, #66]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <16 x i8> %op1 to <16 x i16>
  %2 = sext <16 x i8> %op2 to <16 x i16>
  %mul = mul <16 x i16> %1, %2
  %shr = lshr <16 x i16> %mul, splat (i16 8)
  %res = trunc <16 x i16> %shr to <16 x i8>
  ret <16 x i8> %res
}

define void @smulh_v32i8(ptr %a, ptr %b) {
; SVE-LABEL: smulh_v32i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.b, vl16
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    smulh z2.b, p0/m, z2.b, z3.b
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v32i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    smulh z0.b, z1.b, z0.b
; SVE2-NEXT:    smulh z1.b, z2.b, z3.b
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v32i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #384
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #288] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #304] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #320] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #336] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #352] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #368] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 384
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    mov x29, x0
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
; NONEON-NOSVE-NEXT:    str q1, [sp, #160]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    str q3, [sp, #144]
; NONEON-NOSVE-NEXT:    str q2, [sp, #192]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #184]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #185]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #186]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #187]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #188]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #189]
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #229]
; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #227]
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #228]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #190]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #191]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #176]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #177]
; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #226]
; NONEON-NOSVE-NEXT:    ldrsb w2, [sp, #214]
; NONEON-NOSVE-NEXT:    ldrsb w1, [sp, #215]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #178]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #179]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrsb w4, [sp, #212]
; NONEON-NOSVE-NEXT:    ldrsb w3, [sp, #213]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #180]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #181]
; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #247]
; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #246]
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #244]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #182]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #183]
; NONEON-NOSVE-NEXT:    mul w26, w12, w16
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #242]
; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #250]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #232]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #233]
; NONEON-NOSVE-NEXT:    mul w30, w10, w12
; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #255]
; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #253]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #234]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #235]
; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #249]
; NONEON-NOSVE-NEXT:    ldrsb w6, [sp, #210]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #236]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #237]
; NONEON-NOSVE-NEXT:    ldrsb w5, [sp, #211]
; NONEON-NOSVE-NEXT:    ldrsb w19, [sp, #208]
; NONEON-NOSVE-NEXT:    ldrsb w7, [sp, #209]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #238]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #239]
; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #222]
; NONEON-NOSVE-NEXT:    ldrsb w20, [sp, #223]
; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #220]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #224]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #225]
; NONEON-NOSVE-NEXT:    ldrsb w22, [sp, #221]
; NONEON-NOSVE-NEXT:    ldrsb w24, [sp, #219]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #230]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #231]
; NONEON-NOSVE-NEXT:    mul w27, w8, w14
; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #245]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #217]
; NONEON-NOSVE-NEXT:    mul w9, w9, w15
; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #251]
; NONEON-NOSVE-NEXT:    mul w25, w13, w14
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #243]
; NONEON-NOSVE-NEXT:    lsr w14, w27, #8
; NONEON-NOSVE-NEXT:    ldrsb w27, [sp, #218]
; NONEON-NOSVE-NEXT:    lsr w17, w9, #8
; NONEON-NOSVE-NEXT:    mul w28, w11, w13
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #216]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #287]
; NONEON-NOSVE-NEXT:    lsr w14, w25, #8
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Reload
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #241]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #240]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #285]
; NONEON-NOSVE-NEXT:    lsr w14, w28, #8
; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #254]
; NONEON-NOSVE-NEXT:    mul w8, w25, w8
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #28] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #252]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #283]
; NONEON-NOSVE-NEXT:    ldr w14, [sp, #40] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w9, w25, w9
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #32] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w17, [sp, #286]
; NONEON-NOSVE-NEXT:    mul w12, w14, w12
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    lsr w17, w26, #8
; NONEON-NOSVE-NEXT:    mul w10, w25, w10
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldr w14, [sp, #44] // 4-byte Reload
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #281]
; NONEON-NOSVE-NEXT:    mul w11, w25, w11
; NONEON-NOSVE-NEXT:    strb w17, [sp, #284]
; NONEON-NOSVE-NEXT:    lsr w17, w30, #8
; NONEON-NOSVE-NEXT:    mul w13, w14, w13
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #48] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #280]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #320] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    mul w10, w10, w15
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #52] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #279]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #56] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w11, w11, w16
; NONEON-NOSVE-NEXT:    strb w9, [sp, #278]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    mul w12, w12, w18
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #60] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #277]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #64] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #276]
; NONEON-NOSVE-NEXT:    mul w13, w13, w0
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #68] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w10, w10, w1
; NONEON-NOSVE-NEXT:    strb w8, [sp, #275]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w2
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #72] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #274]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #76] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #273]
; NONEON-NOSVE-NEXT:    mul w12, w12, w3
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #80] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w13, w13, w4
; NONEON-NOSVE-NEXT:    strb w9, [sp, #272]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    mul w10, w10, w5
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #84] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #271]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #88] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #270]
; NONEON-NOSVE-NEXT:    mul w11, w11, w6
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #92] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w12, w12, w7
; NONEON-NOSVE-NEXT:    strb w8, [sp, #269]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    mul w13, w13, w19
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #96] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #268]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #100] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #267]
; NONEON-NOSVE-NEXT:    mul w10, w10, w20
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #104] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w11, w11, w21
; NONEON-NOSVE-NEXT:    strb w9, [sp, #266]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #108] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w12, w12, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #265]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #112] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #264]
; NONEON-NOSVE-NEXT:    mul w13, w13, w23
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #116] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldp w15, w14, [sp, #16] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    mul w10, w10, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #263]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w27
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #120] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #262]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #124] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #261]
; NONEON-NOSVE-NEXT:    mul w12, w12, w15
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    strb w17, [sp, #282]
; NONEON-NOSVE-NEXT:    mul w13, w13, w14
; NONEON-NOSVE-NEXT:    strb w9, [sp, #260]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #259]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #368] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    strb w9, [sp, #258]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #352] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #257]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #336] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #256]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #304] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x29]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    add sp, sp, #384
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <32 x i8>, ptr %a
  %op2 = load <32 x i8>, ptr %b
  %1 = sext <32 x i8> %op1 to <32 x i16>
  %2 = sext <32 x i8> %op2 to <32 x i16>
  %mul = mul <32 x i16> %1, %2
  %shr = lshr <32 x i16> %mul, splat (i16 8)
  %res = trunc <32 x i16> %shr to <32 x i8>
  store <32 x i8> %res, ptr %a
  ret void
}

define <2 x i16> @smulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
; SVE-LABEL: smulh_v2i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s, vl2
; SVE-NEXT:    sxth z0.s, p0/m, z0.s
; SVE-NEXT:    sxth z1.s, p0/m, z1.s
; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    lsr z0.s, z0.s, #16
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v2i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ptrue p0.s, vl2
; SVE2-NEXT:    sxth z0.s, p0/m, z0.s
; SVE2-NEXT:    sxth z1.s, p0/m, z1.s
; SVE2-NEXT:    mul z0.s, z0.s, z1.s
; SVE2-NEXT:    lsr z0.s, z0.s, #16
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v2i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w8, w8, w10
; NONEON-NOSVE-NEXT:    mul w9, w9, w11
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <2 x i16> %op1 to <2 x i32>
  %2 = sext <2 x i16> %op2 to <2 x i32>
  %mul = mul <2 x i32> %1, %2
  %shr = lshr <2 x i32> %mul, splat (i32 16)
  %res = trunc <2 x i32> %shr to <2 x i16>
  ret <2 x i16> %res
}

define <4 x i16> @smulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
; SVE-LABEL: smulh_v4i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v4i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.h, z0.h, z1.h
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v4i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #18]
; NONEON-NOSVE-NEXT:    mul w11, w11, w12
; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w10, w10, w13
; NONEON-NOSVE-NEXT:    mul w9, w9, w14
; NONEON-NOSVE-NEXT:    mul w8, w8, w12
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    strh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    strh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <4 x i16> %op1 to <4 x i32>
  %2 = sext <4 x i16> %op2 to <4 x i32>
  %mul = mul <4 x i32> %1, %2
  %shr = lshr <4 x i32> %mul, splat (i32 16)
  %res = trunc <4 x i32> %shr to <4 x i16>
  ret <4 x i16> %res
}

define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
; SVE-LABEL: smulh_v8i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v8i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.h, z0.h, z1.h
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v8i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsh w15, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #42]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrsh w0, [sp, #62]
; NONEON-NOSVE-NEXT:    mul w15, w15, w16
; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #48]
; NONEON-NOSVE-NEXT:    mul w14, w14, w17
; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #56]
; NONEON-NOSVE-NEXT:    mul w13, w13, w18
; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #60]
; NONEON-NOSVE-NEXT:    mul w12, w12, w16
; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #58]
; NONEON-NOSVE-NEXT:    lsr w15, w15, #16
; NONEON-NOSVE-NEXT:    mul w11, w11, w0
; NONEON-NOSVE-NEXT:    lsr w14, w14, #16
; NONEON-NOSVE-NEXT:    mul w10, w10, w18
; NONEON-NOSVE-NEXT:    lsr w13, w13, #16
; NONEON-NOSVE-NEXT:    strh w15, [sp, #78]
; NONEON-NOSVE-NEXT:    mul w9, w9, w16
; NONEON-NOSVE-NEXT:    lsr w12, w12, #16
; NONEON-NOSVE-NEXT:    strh w14, [sp, #76]
; NONEON-NOSVE-NEXT:    mul w8, w8, w17
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    strh w13, [sp, #74]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    strh w12, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    strh w11, [sp, #70]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    strh w10, [sp, #68]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #66]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <8 x i16> %op1 to <8 x i32>
  %2 = sext <8 x i16> %op2 to <8 x i32>
  %mul = mul <8 x i32> %1, %2
  %shr = lshr <8 x i32> %mul, splat (i32 16)
  %res = trunc <8 x i32> %shr to <8 x i16>
  ret <8 x i16> %res
}

define void @smulh_v16i16(ptr %a, ptr %b) {
; SVE-LABEL: smulh_v16i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    smulh z2.h, p0/m, z2.h, z3.h
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v16i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    smulh z0.h, z1.h, z0.h
; SVE2-NEXT:    smulh z1.h, z2.h, z3.h
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v16i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #240
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #160] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #176] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #192] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #208] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #224] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 240
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    str q1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    str q3, [sp, #16]
; NONEON-NOSVE-NEXT:    str q2, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #50]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsh w7, [sp, #96]
; NONEON-NOSVE-NEXT:    ldrsh w19, [sp, #98]
; NONEON-NOSVE-NEXT:    ldrsh w20, [sp, #100]
; NONEON-NOSVE-NEXT:    ldrsh w21, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #54]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w2, [sp, #106]
; NONEON-NOSVE-NEXT:    ldrsh w4, [sp, #108]
; NONEON-NOSVE-NEXT:    ldrsh w5, [sp, #110]
; NONEON-NOSVE-NEXT:    ldrsh w15, [sp, #88]
; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrsh w1, [sp, #92]
; NONEON-NOSVE-NEXT:    ldrsh w3, [sp, #94]
; NONEON-NOSVE-NEXT:    mul w8, w8, w15
; NONEON-NOSVE-NEXT:    ldrsh w6, [sp, #80]
; NONEON-NOSVE-NEXT:    ldrsh w23, [sp, #82]
; NONEON-NOSVE-NEXT:    mul w11, w11, w3
; NONEON-NOSVE-NEXT:    ldrsh w25, [sp, #84]
; NONEON-NOSVE-NEXT:    mul w13, w13, w23
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    mul w14, w14, w25
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    mul w12, w12, w6
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
; NONEON-NOSVE-NEXT:    mul w10, w10, w1
; NONEON-NOSVE-NEXT:    lsr w13, w13, #16
; NONEON-NOSVE-NEXT:    ldrsh w22, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrsh w24, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrsh w26, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrsh w27, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsh w28, [sp, #126]
; NONEON-NOSVE-NEXT:    mul w9, w9, w17
; NONEON-NOSVE-NEXT:    mul w21, w21, w22
; NONEON-NOSVE-NEXT:    ldrsh w22, [sp, #86]
; NONEON-NOSVE-NEXT:    lsr w14, w14, #16
; NONEON-NOSVE-NEXT:    mul w20, w20, w24
; NONEON-NOSVE-NEXT:    ldrsh w24, [sp, #120]
; NONEON-NOSVE-NEXT:    lsr w12, w12, #16
; NONEON-NOSVE-NEXT:    mul w19, w19, w26
; NONEON-NOSVE-NEXT:    ldrsh w26, [sp, #124]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    mul w7, w7, w27
; NONEON-NOSVE-NEXT:    ldrsh w27, [sp, #122]
; NONEON-NOSVE-NEXT:    lsr w21, w21, #16
; NONEON-NOSVE-NEXT:    mul w5, w5, w28
; NONEON-NOSVE-NEXT:    lsr w20, w20, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    mul w4, w4, w26
; NONEON-NOSVE-NEXT:    lsr w19, w19, #16
; NONEON-NOSVE-NEXT:    strh w21, [sp, #158]
; NONEON-NOSVE-NEXT:    mul w2, w2, w27
; NONEON-NOSVE-NEXT:    lsr w7, w7, #16
; NONEON-NOSVE-NEXT:    strh w20, [sp, #156]
; NONEON-NOSVE-NEXT:    mul w18, w18, w24
; NONEON-NOSVE-NEXT:    lsr w5, w5, #16
; NONEON-NOSVE-NEXT:    strh w19, [sp, #154]
; NONEON-NOSVE-NEXT:    mul w16, w16, w22
; NONEON-NOSVE-NEXT:    lsr w4, w4, #16
; NONEON-NOSVE-NEXT:    strh w7, [sp, #152]
; NONEON-NOSVE-NEXT:    lsr w2, w2, #16
; NONEON-NOSVE-NEXT:    strh w5, [sp, #150]
; NONEON-NOSVE-NEXT:    lsr w18, w18, #16
; NONEON-NOSVE-NEXT:    strh w4, [sp, #148]
; NONEON-NOSVE-NEXT:    lsr w16, w16, #16
; NONEON-NOSVE-NEXT:    strh w2, [sp, #146]
; NONEON-NOSVE-NEXT:    strh w18, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #224] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w16, [sp, #142]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #208] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w14, [sp, #140]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #192] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w13, [sp, #138]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #176] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w12, [sp, #136]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #160] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w11, [sp, #134]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #132]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #130]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #240
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <16 x i16>, ptr %a
  %op2 = load <16 x i16>, ptr %b
  %1 = sext <16 x i16> %op1 to <16 x i32>
  %2 = sext <16 x i16> %op2 to <16 x i32>
  %mul = mul <16 x i32> %1, %2
  %shr = lshr <16 x i32> %mul, splat (i32 16)
  %res = trunc <16 x i32> %shr to <16 x i16>
  store <16 x i16> %res, ptr %a
  ret void
}

define <2 x i32> @smulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
; SVE-LABEL: smulh_v2i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s, vl2
; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v2i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.s, z0.s, z1.s
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v2i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #8]
; NONEON-NOSVE-NEXT:    ldpsw x11, x10, [sp, #16]
; NONEON-NOSVE-NEXT:    smull x9, w9, w10
; NONEON-NOSVE-NEXT:    smull x8, w8, w11
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <2 x i32> %op1 to <2 x i64>
  %2 = sext <2 x i32> %op2 to <2 x i64>
  %mul = mul <2 x i64> %1, %2
  %shr = lshr <2 x i64> %mul, splat (i64 32)
  %res = trunc <2 x i64> %shr to <2 x i32>
  ret <2 x i32> %res
}

define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
; SVE-LABEL: smulh_v4i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s, vl4
; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v4i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.s, z0.s, z1.s
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v4i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldpsw x10, x11, [sp, #32]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldpsw x13, x12, [sp, #48]
; NONEON-NOSVE-NEXT:    smull x11, w11, w12
; NONEON-NOSVE-NEXT:    ldpsw x14, x12, [sp, #56]
; NONEON-NOSVE-NEXT:    smull x10, w10, w13
; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
; NONEON-NOSVE-NEXT:    smull x9, w9, w12
; NONEON-NOSVE-NEXT:    smull x8, w8, w14
; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <4 x i32> %op1 to <4 x i64>
  %2 = sext <4 x i32> %op2 to <4 x i64>
  %mul = mul <4 x i64> %1, %2
  %shr = lshr <4 x i64> %mul, splat (i64 32)
  %res = trunc <4 x i64> %shr to <4 x i32>
  ret <4 x i32> %res
}

define void @smulh_v8i32(ptr %a, ptr %b) {
; SVE-LABEL: smulh_v8i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.s, vl4
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    smulh z2.s, p0/m, z2.s, z3.s
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v8i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    smulh z0.s, z1.s, z0.s
; SVE2-NEXT:    smulh z1.s, z2.s, z3.s
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q0, q3, [sp]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldpsw x10, x11, [sp, #48]
; NONEON-NOSVE-NEXT:    ldpsw x12, x13, [sp, #104]
; NONEON-NOSVE-NEXT:    ldpsw x14, x15, [sp, #96]
; NONEON-NOSVE-NEXT:    str q2, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
; NONEON-NOSVE-NEXT:    ldpsw x17, x16, [sp, #112]
; NONEON-NOSVE-NEXT:    smull x15, w15, w16
; NONEON-NOSVE-NEXT:    ldpsw x18, x16, [sp, #120]
; NONEON-NOSVE-NEXT:    smull x14, w14, w17
; NONEON-NOSVE-NEXT:    ldpsw x17, x1, [sp, #80]
; NONEON-NOSVE-NEXT:    smull x13, w13, w16
; NONEON-NOSVE-NEXT:    lsr x15, x15, #32
; NONEON-NOSVE-NEXT:    smull x12, w12, w18
; NONEON-NOSVE-NEXT:    lsr x14, x14, #32
; NONEON-NOSVE-NEXT:    ldpsw x16, x18, [sp, #88]
; NONEON-NOSVE-NEXT:    smull x11, w11, w1
; NONEON-NOSVE-NEXT:    lsr x13, x13, #32
; NONEON-NOSVE-NEXT:    stp w14, w15, [sp, #152]
; NONEON-NOSVE-NEXT:    smull x10, w10, w17
; NONEON-NOSVE-NEXT:    lsr x12, x12, #32
; NONEON-NOSVE-NEXT:    smull x9, w9, w18
; NONEON-NOSVE-NEXT:    smull x8, w8, w16
; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
; NONEON-NOSVE-NEXT:    stp w12, w13, [sp, #144]
; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <8 x i32>, ptr %a
  %op2 = load <8 x i32>, ptr %b
  %1 = sext <8 x i32> %op1 to <8 x i64>
  %2 = sext <8 x i32> %op2 to <8 x i64>
  %mul = mul <8 x i64> %1, %2
  %shr = lshr <8 x i64> %mul,  splat (i64 32)
  %res = trunc <8 x i64> %shr to <8 x i32>
  store <8 x i32> %res, ptr %a
  ret void
}

define <1 x i64> @smulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
; SVE-LABEL: smulh_v1i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d, vl1
; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v1i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.d, z0.d, z1.d
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v1i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #16
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT:    fmov x8, d0
; NONEON-NOSVE-NEXT:    fmov x9, d1
; NONEON-NOSVE-NEXT:    smulh x8, x8, x9
; NONEON-NOSVE-NEXT:    fmov d0, x8
; NONEON-NOSVE-NEXT:    add sp, sp, #16
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <1 x i64> %op1 to <1 x i128>
  %2 = sext <1 x i64> %op2 to <1 x i128>
  %mul = mul <1 x i128> %1, %2
  %shr = lshr <1 x i128> %mul, splat (i128 64)
  %res = trunc <1 x i128> %shr to <1 x i64>
  ret <1 x i64> %res
}

define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
; SVE-LABEL: smulh_v2i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d, vl2
; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v2i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    smulh z0.d, z0.d, z1.d
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v2i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp]
; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp, #16]
; NONEON-NOSVE-NEXT:    smulh x8, x8, x10
; NONEON-NOSVE-NEXT:    smulh x9, x9, x11
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %1 = sext <2 x i64> %op1 to <2 x i128>
  %2 = sext <2 x i64> %op2 to <2 x i128>
  %mul = mul <2 x i128> %1, %2
  %shr = lshr <2 x i128> %mul, splat (i128 64)
  %res = trunc <2 x i128> %shr to <2 x i64>
  ret <2 x i64> %res
}

define void @smulh_v4i64(ptr %a, ptr %b) {
; SVE-LABEL: smulh_v4i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.d, vl2
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    smulh z2.d, p0/m, z2.d, z3.d
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: smulh_v4i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    smulh z0.d, z1.d, z0.d
; SVE2-NEXT:    smulh z1.d, z2.d, z3.d
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: smulh_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #128
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q2, [sp]
; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp]
; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp x13, x12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT:    smulh x10, x10, x12
; NONEON-NOSVE-NEXT:    ldp x14, x12, [sp, #48]
; NONEON-NOSVE-NEXT:    smulh x11, x11, x13
; NONEON-NOSVE-NEXT:    smulh x8, x8, x12
; NONEON-NOSVE-NEXT:    smulh x9, x9, x14
; NONEON-NOSVE-NEXT:    stp x11, x10, [sp, #96]
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #128
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <4 x i64>, ptr %a
  %op2 = load <4 x i64>, ptr %b
  %1 = sext <4 x i64> %op1 to <4 x i128>
  %2 = sext <4 x i64> %op2 to <4 x i128>
  %mul = mul <4 x i128> %1, %2
  %shr = lshr <4 x i128> %mul, splat (i128 64)
  %res = trunc <4 x i128> %shr to <4 x i64>
  store <4 x i64> %res, ptr %a
  ret void
}

;
; UMULH
;

define <4 x i8> @umulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
; SVE-LABEL: umulh_v4i8:
; SVE:       // %bb.0:
; SVE-NEXT:    and z0.h, z0.h, #0xff
; SVE-NEXT:    and z1.h, z1.h, #0xff
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    lsr z0.h, z0.h, #4
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v4i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    and z0.h, z0.h, #0xff
; SVE2-NEXT:    and z1.h, z1.h, #0xff
; SVE2-NEXT:    mul z0.h, z0.h, z1.h
; SVE2-NEXT:    lsr z0.h, z0.h, #4
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v4i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #18]
; NONEON-NOSVE-NEXT:    mul w8, w8, w12
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w9, w9, w13
; NONEON-NOSVE-NEXT:    mul w10, w10, w14
; NONEON-NOSVE-NEXT:    mul w11, w11, w12
; NONEON-NOSVE-NEXT:    lsr w8, w8, #4
; NONEON-NOSVE-NEXT:    lsr w9, w9, #4
; NONEON-NOSVE-NEXT:    lsr w10, w10, #4
; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
; NONEON-NOSVE-NEXT:    lsr w8, w11, #4
; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <4 x i8> %op1 to <4 x i16>
  %2 = zext <4 x i8> %op2 to <4 x i16>
  %mul = mul <4 x i16> %1, %2
  %shr = lshr <4 x i16> %mul, splat (i16 4)
  %res = trunc <4 x i16> %shr to <4 x i8>
  ret <4 x i8> %res
}

define <8 x i8> @umulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
; SVE-LABEL: umulh_v8i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.b, vl8
; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v8i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.b, z0.b, z1.b
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v8i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #15]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #23]
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #13]
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #22]
; NONEON-NOSVE-NEXT:    mul w15, w15, w16
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #21]
; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    mul w14, w14, w17
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #10]
; NONEON-NOSVE-NEXT:    mul w13, w13, w16
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #11]
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w12, w12, w18
; NONEON-NOSVE-NEXT:    lsr w15, w15, #8
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #18]
; NONEON-NOSVE-NEXT:    lsr w14, w14, #8
; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #17]
; NONEON-NOSVE-NEXT:    mul w8, w8, w17
; NONEON-NOSVE-NEXT:    lsr w13, w13, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w0
; NONEON-NOSVE-NEXT:    lsr w12, w12, #8
; NONEON-NOSVE-NEXT:    strb w15, [sp, #31]
; NONEON-NOSVE-NEXT:    mul w10, w10, w16
; NONEON-NOSVE-NEXT:    strb w14, [sp, #30]
; NONEON-NOSVE-NEXT:    mul w9, w9, w18
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    strb w13, [sp, #29]
; NONEON-NOSVE-NEXT:    lsr w11, w11, #8
; NONEON-NOSVE-NEXT:    strb w12, [sp, #28]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    strb w11, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w10, [sp, #26]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <8 x i8> %op1 to <8 x i16>
  %2 = zext <8 x i8> %op2 to <8 x i16>
  %mul = mul <8 x i16> %1, %2
  %shr = lshr <8 x i16> %mul, splat (i16 8)
  %res = trunc <8 x i16> %shr to <8 x i8>
  ret <8 x i8> %res
}

define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
; SVE-LABEL: umulh_v16i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.b, vl16
; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v16i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.b, z0.b, z1.b
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v16i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    str x27, [sp, #80] // 8-byte Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #96] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #112] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #128] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -80
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp]
; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #40]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #45]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    str d0, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #51]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #63]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #60]
; NONEON-NOSVE-NEXT:    str d1, [sp, #88]
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #52]
; NONEON-NOSVE-NEXT:    mul w20, w20, w21
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #53]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #54]
; NONEON-NOSVE-NEXT:    mul w19, w19, w23
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #55]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #40]
; NONEON-NOSVE-NEXT:    mul w7, w7, w25
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #42]
; NONEON-NOSVE-NEXT:    mul w6, w6, w26
; NONEON-NOSVE-NEXT:    lsr w20, w20, #8
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #43]
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #88]
; NONEON-NOSVE-NEXT:    lsr w19, w19, #8
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #89]
; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #90]
; NONEON-NOSVE-NEXT:    lsr w7, w7, #8
; NONEON-NOSVE-NEXT:    ldrb w1, [sp, #91]
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #92]
; NONEON-NOSVE-NEXT:    mul w9, w9, w16
; NONEON-NOSVE-NEXT:    lsr w6, w6, #8
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #93]
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #94]
; NONEON-NOSVE-NEXT:    mul w11, w11, w1
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #95]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #56]
; NONEON-NOSVE-NEXT:    mul w12, w12, w5
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #59]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #58]
; NONEON-NOSVE-NEXT:    mul w15, w15, w24
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #57]
; NONEON-NOSVE-NEXT:    mul w0, w0, w23
; NONEON-NOSVE-NEXT:    lsr w11, w11, #8
; NONEON-NOSVE-NEXT:    mul w4, w4, w27
; NONEON-NOSVE-NEXT:    lsr w12, w12, #8
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    mul w3, w3, w25
; NONEON-NOSVE-NEXT:    lsr w15, w15, #8
; NONEON-NOSVE-NEXT:    strb w20, [sp, #79]
; NONEON-NOSVE-NEXT:    mul w2, w2, w26
; NONEON-NOSVE-NEXT:    lsr w0, w0, #8
; NONEON-NOSVE-NEXT:    strb w19, [sp, #78]
; NONEON-NOSVE-NEXT:    mul w17, w17, w21
; NONEON-NOSVE-NEXT:    lsr w4, w4, #8
; NONEON-NOSVE-NEXT:    strb w7, [sp, #77]
; NONEON-NOSVE-NEXT:    mul w13, w13, w22
; NONEON-NOSVE-NEXT:    lsr w3, w3, #8
; NONEON-NOSVE-NEXT:    strb w6, [sp, #76]
; NONEON-NOSVE-NEXT:    mul w10, w10, w18
; NONEON-NOSVE-NEXT:    lsr w2, w2, #8
; NONEON-NOSVE-NEXT:    strb w4, [sp, #75]
; NONEON-NOSVE-NEXT:    mul w8, w8, w14
; NONEON-NOSVE-NEXT:    lsr w17, w17, #8
; NONEON-NOSVE-NEXT:    strb w3, [sp, #74]
; NONEON-NOSVE-NEXT:    lsr w13, w13, #8
; NONEON-NOSVE-NEXT:    strb w2, [sp, #73]
; NONEON-NOSVE-NEXT:    ldr x27, [sp, #80] // 8-byte Reload
; NONEON-NOSVE-NEXT:    lsr w10, w10, #8
; NONEON-NOSVE-NEXT:    strb w0, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    strb w17, [sp, #71]
; NONEON-NOSVE-NEXT:    strb w15, [sp, #70]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w13, [sp, #69]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #128] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w12, [sp, #68]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #112] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w11, [sp, #67]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #96] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w10, [sp, #66]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <16 x i8> %op1 to <16 x i16>
  %2 = zext <16 x i8> %op2 to <16 x i16>
  %mul = mul <16 x i16> %1, %2
  %shr = lshr <16 x i16> %mul, splat (i16 8)
  %res = trunc <16 x i16> %shr to <16 x i8>
  ret <16 x i8> %res
}

define void @umulh_v32i8(ptr %a, ptr %b) {
; SVE-LABEL: umulh_v32i8:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.b, vl16
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
; SVE-NEXT:    umulh z2.b, p0/m, z2.b, z3.b
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v32i8:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    umulh z0.b, z1.b, z0.b
; SVE2-NEXT:    umulh z1.b, z2.b, z3.b
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v32i8:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #384
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #288] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #304] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #320] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #336] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #352] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #368] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 384
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    mov x29, x0
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
; NONEON-NOSVE-NEXT:    str q1, [sp, #160]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    str q3, [sp, #144]
; NONEON-NOSVE-NEXT:    str q2, [sp, #192]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #184]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #185]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #186]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #187]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #188]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #189]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #229]
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #227]
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #228]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #190]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #191]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #176]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #177]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #226]
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #214]
; NONEON-NOSVE-NEXT:    ldrb w1, [sp, #215]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #179]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #212]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #213]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #180]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #181]
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #247]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #246]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #244]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #183]
; NONEON-NOSVE-NEXT:    mul w26, w12, w16
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #242]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #250]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #232]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #233]
; NONEON-NOSVE-NEXT:    mul w30, w10, w12
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #255]
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #253]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #234]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #235]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #249]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #210]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #236]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #237]
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #211]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #208]
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #209]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #238]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #239]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #222]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #223]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #220]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #224]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #225]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #221]
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #219]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #230]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #231]
; NONEON-NOSVE-NEXT:    mul w27, w8, w14
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #245]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #217]
; NONEON-NOSVE-NEXT:    mul w9, w9, w15
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #251]
; NONEON-NOSVE-NEXT:    mul w25, w13, w14
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #243]
; NONEON-NOSVE-NEXT:    lsr w14, w27, #8
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #218]
; NONEON-NOSVE-NEXT:    lsr w17, w9, #8
; NONEON-NOSVE-NEXT:    mul w28, w11, w13
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #216]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #287]
; NONEON-NOSVE-NEXT:    lsr w14, w25, #8
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Reload
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #241]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #240]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #285]
; NONEON-NOSVE-NEXT:    lsr w14, w28, #8
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #254]
; NONEON-NOSVE-NEXT:    mul w8, w25, w8
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #28] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #252]
; NONEON-NOSVE-NEXT:    strb w14, [sp, #283]
; NONEON-NOSVE-NEXT:    ldr w14, [sp, #40] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w9, w25, w9
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #32] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w17, [sp, #286]
; NONEON-NOSVE-NEXT:    mul w12, w14, w12
; NONEON-NOSVE-NEXT:    lsr w8, w8, #8
; NONEON-NOSVE-NEXT:    lsr w17, w26, #8
; NONEON-NOSVE-NEXT:    mul w10, w25, w10
; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldr w14, [sp, #44] // 4-byte Reload
; NONEON-NOSVE-NEXT:    lsr w9, w9, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #281]
; NONEON-NOSVE-NEXT:    mul w11, w25, w11
; NONEON-NOSVE-NEXT:    strb w17, [sp, #284]
; NONEON-NOSVE-NEXT:    lsr w17, w30, #8
; NONEON-NOSVE-NEXT:    mul w13, w14, w13
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #48] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #280]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #320] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    mul w10, w10, w15
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #52] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #279]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #56] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w11, w11, w16
; NONEON-NOSVE-NEXT:    strb w9, [sp, #278]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    mul w12, w12, w18
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #60] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #277]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #64] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #276]
; NONEON-NOSVE-NEXT:    mul w13, w13, w0
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #68] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w10, w10, w1
; NONEON-NOSVE-NEXT:    strb w8, [sp, #275]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w2
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #72] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #274]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #76] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #273]
; NONEON-NOSVE-NEXT:    mul w12, w12, w3
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #80] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w13, w13, w4
; NONEON-NOSVE-NEXT:    strb w9, [sp, #272]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    mul w10, w10, w5
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #84] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #271]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #88] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #270]
; NONEON-NOSVE-NEXT:    mul w11, w11, w6
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #92] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w12, w12, w7
; NONEON-NOSVE-NEXT:    strb w8, [sp, #269]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    mul w13, w13, w19
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #96] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #268]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #100] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #267]
; NONEON-NOSVE-NEXT:    mul w10, w10, w20
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #104] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w11, w11, w21
; NONEON-NOSVE-NEXT:    strb w9, [sp, #266]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #108] // 4-byte Reload
; NONEON-NOSVE-NEXT:    mul w12, w12, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #265]
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    ldr w10, [sp, #112] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #264]
; NONEON-NOSVE-NEXT:    mul w13, w13, w23
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    ldr w11, [sp, #116] // 4-byte Reload
; NONEON-NOSVE-NEXT:    ldp w15, w14, [sp, #16] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    mul w10, w10, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #263]
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    mul w11, w11, w27
; NONEON-NOSVE-NEXT:    ldr w12, [sp, #120] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #262]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldr w13, [sp, #124] // 4-byte Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #261]
; NONEON-NOSVE-NEXT:    mul w12, w12, w15
; NONEON-NOSVE-NEXT:    lsr w8, w10, #8
; NONEON-NOSVE-NEXT:    strb w17, [sp, #282]
; NONEON-NOSVE-NEXT:    mul w13, w13, w14
; NONEON-NOSVE-NEXT:    strb w9, [sp, #260]
; NONEON-NOSVE-NEXT:    lsr w9, w11, #8
; NONEON-NOSVE-NEXT:    strb w8, [sp, #259]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #368] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    lsr w8, w12, #8
; NONEON-NOSVE-NEXT:    strb w9, [sp, #258]
; NONEON-NOSVE-NEXT:    lsr w9, w13, #8
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #352] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w8, [sp, #257]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #336] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #256]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #304] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x29]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    add sp, sp, #384
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <32 x i8>, ptr %a
  %op2 = load <32 x i8>, ptr %b
  %1 = zext <32 x i8> %op1 to <32 x i16>
  %2 = zext <32 x i8> %op2 to <32 x i16>
  %mul = mul <32 x i16> %1, %2
  %shr = lshr <32 x i16> %mul, splat (i16 8)
  %res = trunc <32 x i16> %shr to <32 x i8>
  store <32 x i8> %res, ptr %a
  ret void
}

define <2 x i16> @umulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
; SVE-LABEL: umulh_v2i16:
; SVE:       // %bb.0:
; SVE-NEXT:    and z0.s, z0.s, #0xffff
; SVE-NEXT:    and z1.s, z1.s, #0xffff
; SVE-NEXT:    ptrue p0.s, vl2
; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    lsr z0.s, z0.s, #16
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v2i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    and z0.s, z0.s, #0xffff
; SVE2-NEXT:    and z1.s, z1.s, #0xffff
; SVE2-NEXT:    mul z0.s, z0.s, z1.s
; SVE2-NEXT:    lsr z0.s, z0.s, #16
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v2i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w8, w8, w10
; NONEON-NOSVE-NEXT:    mul w9, w9, w11
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <2 x i16> %op1 to <2 x i32>
  %2 = zext <2 x i16> %op2 to <2 x i32>
  %mul = mul <2 x i32> %1, %2
  %shr = lshr <2 x i32> %mul, splat (i32 16)
  %res = trunc <2 x i32> %shr to <2 x i16>
  ret <2 x i16> %res
}

define <4 x i16> @umulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
; SVE-LABEL: umulh_v4i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v4i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.h, z0.h, z1.h
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v4i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #18]
; NONEON-NOSVE-NEXT:    mul w11, w11, w12
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    mul w10, w10, w13
; NONEON-NOSVE-NEXT:    mul w9, w9, w14
; NONEON-NOSVE-NEXT:    mul w8, w8, w12
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    strh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    strh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <4 x i16> %op1 to <4 x i32>
  %2 = zext <4 x i16> %op2 to <4 x i32>
  %mul = mul <4 x i32> %1, %2
  %shr = lshr <4 x i32> %mul, splat (i32 16)
  %res = trunc <4 x i32> %shr to <4 x i16>
  ret <4 x i16> %res
}

define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
; SVE-LABEL: umulh_v8i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v8i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.h, z0.h, z1.h
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v8i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #62]
; NONEON-NOSVE-NEXT:    mul w15, w15, w16
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #48]
; NONEON-NOSVE-NEXT:    mul w14, w14, w17
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #56]
; NONEON-NOSVE-NEXT:    mul w13, w13, w18
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #60]
; NONEON-NOSVE-NEXT:    mul w12, w12, w16
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #58]
; NONEON-NOSVE-NEXT:    lsr w15, w15, #16
; NONEON-NOSVE-NEXT:    mul w11, w11, w0
; NONEON-NOSVE-NEXT:    lsr w14, w14, #16
; NONEON-NOSVE-NEXT:    mul w10, w10, w18
; NONEON-NOSVE-NEXT:    lsr w13, w13, #16
; NONEON-NOSVE-NEXT:    strh w15, [sp, #78]
; NONEON-NOSVE-NEXT:    mul w9, w9, w16
; NONEON-NOSVE-NEXT:    lsr w12, w12, #16
; NONEON-NOSVE-NEXT:    strh w14, [sp, #76]
; NONEON-NOSVE-NEXT:    mul w8, w8, w17
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    strh w13, [sp, #74]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    strh w12, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    strh w11, [sp, #70]
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    strh w10, [sp, #68]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #66]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <8 x i16> %op1 to <8 x i32>
  %2 = zext <8 x i16> %op2 to <8 x i32>
  %mul = mul <8 x i32> %1, %2
  %shr = lshr <8 x i32> %mul, splat (i32 16)
  %res = trunc <8 x i32> %shr to <8 x i16>
  ret <8 x i16> %res
}

define void @umulh_v16i16(ptr %a, ptr %b) {
; SVE-LABEL: umulh_v16i16:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    umulh z2.h, p0/m, z2.h, z3.h
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v16i16:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    umulh z0.h, z1.h, z0.h
; SVE2-NEXT:    umulh z1.h, z2.h, z3.h
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v16i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #240
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #160] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #176] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #192] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #208] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #224] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 240
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    str q1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    str q3, [sp, #16]
; NONEON-NOSVE-NEXT:    str q2, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #50]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w7, [sp, #96]
; NONEON-NOSVE-NEXT:    ldrh w19, [sp, #98]
; NONEON-NOSVE-NEXT:    ldrh w20, [sp, #100]
; NONEON-NOSVE-NEXT:    ldrh w21, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #54]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #106]
; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #108]
; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #110]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #88]
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrh w1, [sp, #92]
; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #94]
; NONEON-NOSVE-NEXT:    mul w8, w8, w15
; NONEON-NOSVE-NEXT:    ldrh w6, [sp, #80]
; NONEON-NOSVE-NEXT:    ldrh w23, [sp, #82]
; NONEON-NOSVE-NEXT:    mul w11, w11, w3
; NONEON-NOSVE-NEXT:    ldrh w25, [sp, #84]
; NONEON-NOSVE-NEXT:    mul w13, w13, w23
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    mul w14, w14, w25
; NONEON-NOSVE-NEXT:    lsr w8, w8, #16
; NONEON-NOSVE-NEXT:    mul w12, w12, w6
; NONEON-NOSVE-NEXT:    lsr w11, w11, #16
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
; NONEON-NOSVE-NEXT:    mul w10, w10, w1
; NONEON-NOSVE-NEXT:    lsr w13, w13, #16
; NONEON-NOSVE-NEXT:    ldrh w22, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrh w24, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrh w26, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrh w27, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrh w28, [sp, #126]
; NONEON-NOSVE-NEXT:    mul w9, w9, w17
; NONEON-NOSVE-NEXT:    mul w21, w21, w22
; NONEON-NOSVE-NEXT:    ldrh w22, [sp, #86]
; NONEON-NOSVE-NEXT:    lsr w14, w14, #16
; NONEON-NOSVE-NEXT:    mul w20, w20, w24
; NONEON-NOSVE-NEXT:    ldrh w24, [sp, #120]
; NONEON-NOSVE-NEXT:    lsr w12, w12, #16
; NONEON-NOSVE-NEXT:    mul w19, w19, w26
; NONEON-NOSVE-NEXT:    ldrh w26, [sp, #124]
; NONEON-NOSVE-NEXT:    lsr w10, w10, #16
; NONEON-NOSVE-NEXT:    mul w7, w7, w27
; NONEON-NOSVE-NEXT:    ldrh w27, [sp, #122]
; NONEON-NOSVE-NEXT:    lsr w21, w21, #16
; NONEON-NOSVE-NEXT:    mul w5, w5, w28
; NONEON-NOSVE-NEXT:    lsr w20, w20, #16
; NONEON-NOSVE-NEXT:    lsr w9, w9, #16
; NONEON-NOSVE-NEXT:    mul w4, w4, w26
; NONEON-NOSVE-NEXT:    lsr w19, w19, #16
; NONEON-NOSVE-NEXT:    strh w21, [sp, #158]
; NONEON-NOSVE-NEXT:    mul w2, w2, w27
; NONEON-NOSVE-NEXT:    lsr w7, w7, #16
; NONEON-NOSVE-NEXT:    strh w20, [sp, #156]
; NONEON-NOSVE-NEXT:    mul w18, w18, w24
; NONEON-NOSVE-NEXT:    lsr w5, w5, #16
; NONEON-NOSVE-NEXT:    strh w19, [sp, #154]
; NONEON-NOSVE-NEXT:    mul w16, w16, w22
; NONEON-NOSVE-NEXT:    lsr w4, w4, #16
; NONEON-NOSVE-NEXT:    strh w7, [sp, #152]
; NONEON-NOSVE-NEXT:    lsr w2, w2, #16
; NONEON-NOSVE-NEXT:    strh w5, [sp, #150]
; NONEON-NOSVE-NEXT:    lsr w18, w18, #16
; NONEON-NOSVE-NEXT:    strh w4, [sp, #148]
; NONEON-NOSVE-NEXT:    lsr w16, w16, #16
; NONEON-NOSVE-NEXT:    strh w2, [sp, #146]
; NONEON-NOSVE-NEXT:    strh w18, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #224] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w16, [sp, #142]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #208] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w14, [sp, #140]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #192] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w13, [sp, #138]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #176] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w12, [sp, #136]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #160] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w11, [sp, #134]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #132]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #130]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #240
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <16 x i16>, ptr %a
  %op2 = load <16 x i16>, ptr %b
  %1 = zext <16 x i16> %op1 to <16 x i32>
  %2 = zext <16 x i16> %op2 to <16 x i32>
  %mul = mul <16 x i32> %1, %2
  %shr = lshr <16 x i32> %mul, splat (i32 16)
  %res = trunc <16 x i32> %shr to <16 x i16>
  store <16 x i16> %res, ptr %a
  ret void
}

define <2 x i32> @umulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
; SVE-LABEL: umulh_v2i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s, vl2
; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v2i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.s, z0.s, z1.s
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v2i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #32
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #8]
; NONEON-NOSVE-NEXT:    ldp w11, w10, [sp, #16]
; NONEON-NOSVE-NEXT:    umull x9, w9, w10
; NONEON-NOSVE-NEXT:    umull x8, w8, w11
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <2 x i32> %op1 to <2 x i64>
  %2 = zext <2 x i32> %op2 to <2 x i64>
  %mul = mul <2 x i64> %1, %2
  %shr = lshr <2 x i64> %mul, splat (i64 32)
  %res = trunc <2 x i64> %shr to <2 x i32>
  ret <2 x i32> %res
}

define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
; SVE-LABEL: umulh_v4i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s, vl4
; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v4i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.s, z0.s, z1.s
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v4i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp w13, w12, [sp, #48]
; NONEON-NOSVE-NEXT:    umull x11, w11, w12
; NONEON-NOSVE-NEXT:    ldp w14, w12, [sp, #56]
; NONEON-NOSVE-NEXT:    umull x10, w10, w13
; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
; NONEON-NOSVE-NEXT:    umull x9, w9, w12
; NONEON-NOSVE-NEXT:    umull x8, w8, w14
; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <4 x i32> %op1 to <4 x i64>
  %2 = zext <4 x i32> %op2 to <4 x i64>
  %mul = mul <4 x i64> %1, %2
  %shr = lshr <4 x i64> %mul, splat (i64 32)
  %res = trunc <4 x i64> %shr to <4 x i32>
  ret <4 x i32> %res
}

define void @umulh_v8i32(ptr %a, ptr %b) {
; SVE-LABEL: umulh_v8i32:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.s, vl4
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    umulh z2.s, p0/m, z2.s, z3.s
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v8i32:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    umulh z0.s, z1.s, z0.s
; SVE2-NEXT:    umulh z1.s, z2.s, z3.s
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
; NONEON-NOSVE-NEXT:    str q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q0, q3, [sp]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    str q2, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #48]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #104]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp w17, w16, [sp, #112]
; NONEON-NOSVE-NEXT:    umull x15, w15, w16
; NONEON-NOSVE-NEXT:    ldp w18, w16, [sp, #120]
; NONEON-NOSVE-NEXT:    umull x14, w14, w17
; NONEON-NOSVE-NEXT:    ldp w17, w1, [sp, #80]
; NONEON-NOSVE-NEXT:    umull x13, w13, w16
; NONEON-NOSVE-NEXT:    lsr x15, x15, #32
; NONEON-NOSVE-NEXT:    umull x12, w12, w18
; NONEON-NOSVE-NEXT:    lsr x14, x14, #32
; NONEON-NOSVE-NEXT:    ldp w16, w18, [sp, #88]
; NONEON-NOSVE-NEXT:    umull x11, w11, w1
; NONEON-NOSVE-NEXT:    lsr x13, x13, #32
; NONEON-NOSVE-NEXT:    stp w14, w15, [sp, #152]
; NONEON-NOSVE-NEXT:    umull x10, w10, w17
; NONEON-NOSVE-NEXT:    lsr x12, x12, #32
; NONEON-NOSVE-NEXT:    umull x9, w9, w18
; NONEON-NOSVE-NEXT:    umull x8, w8, w16
; NONEON-NOSVE-NEXT:    lsr x11, x11, #32
; NONEON-NOSVE-NEXT:    stp w12, w13, [sp, #144]
; NONEON-NOSVE-NEXT:    lsr x10, x10, #32
; NONEON-NOSVE-NEXT:    lsr x9, x9, #32
; NONEON-NOSVE-NEXT:    lsr x8, x8, #32
; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <8 x i32>, ptr %a
  %op2 = load <8 x i32>, ptr %b
  %1 = zext <8 x i32> %op1 to <8 x i64>
  %2 = zext <8 x i32> %op2 to <8 x i64>
  %mul = mul <8 x i64> %1, %2
  %shr = lshr <8 x i64> %mul, splat (i64 32)
  %res = trunc <8 x i64> %shr to <8 x i32>
  store <8 x i32> %res, ptr %a
  ret void
}

define <1 x i64> @umulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
; SVE-LABEL: umulh_v1i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d, vl1
; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v1i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.d, z0.d, z1.d
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v1i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #16
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT:    fmov x8, d0
; NONEON-NOSVE-NEXT:    fmov x9, d1
; NONEON-NOSVE-NEXT:    umulh x8, x8, x9
; NONEON-NOSVE-NEXT:    fmov d0, x8
; NONEON-NOSVE-NEXT:    add sp, sp, #16
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <1 x i64> %op1 to <1 x i128>
  %2 = zext <1 x i64> %op2 to <1 x i128>
  %mul = mul <1 x i128> %1, %2
  %shr = lshr <1 x i128> %mul, splat (i128 64)
  %res = trunc <1 x i128> %shr to <1 x i64>
  ret <1 x i64> %res
}

define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
; SVE-LABEL: umulh_v2i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d, vl2
; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v2i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    umulh z0.d, z0.d, z1.d
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v2i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp]
; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp, #16]
; NONEON-NOSVE-NEXT:    umulh x8, x8, x10
; NONEON-NOSVE-NEXT:    umulh x9, x9, x11
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %1 = zext <2 x i64> %op1 to <2 x i128>
  %2 = zext <2 x i64> %op2 to <2 x i128>
  %mul = mul <2 x i128> %1, %2
  %shr = lshr <2 x i128> %mul, splat (i128 64)
  %res = trunc <2 x i128> %shr to <2 x i64>
  ret <2 x i64> %res
}

define void @umulh_v4i64(ptr %a, ptr %b) {
; SVE-LABEL: umulh_v4i64:
; SVE:       // %bb.0:
; SVE-NEXT:    ldp q0, q3, [x1]
; SVE-NEXT:    ptrue p0.d, vl2
; SVE-NEXT:    ldp q1, q2, [x0]
; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    umulh z2.d, p0/m, z2.d, z3.d
; SVE-NEXT:    stp q0, q2, [x0]
; SVE-NEXT:    ret
;
; SVE2-LABEL: umulh_v4i64:
; SVE2:       // %bb.0:
; SVE2-NEXT:    ldp q0, q3, [x1]
; SVE2-NEXT:    ldp q1, q2, [x0]
; SVE2-NEXT:    umulh z0.d, z1.d, z0.d
; SVE2-NEXT:    umulh z1.d, z2.d, z3.d
; SVE2-NEXT:    stp q0, q1, [x0]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: umulh_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #128
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q2, [sp]
; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp]
; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp x13, x12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT:    umulh x10, x10, x12
; NONEON-NOSVE-NEXT:    ldp x14, x12, [sp, #48]
; NONEON-NOSVE-NEXT:    umulh x11, x11, x13
; NONEON-NOSVE-NEXT:    umulh x8, x8, x12
; NONEON-NOSVE-NEXT:    umulh x9, x9, x14
; NONEON-NOSVE-NEXT:    stp x11, x10, [sp, #96]
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #128
; NONEON-NOSVE-NEXT:    ret
  %op1 = load <4 x i64>, ptr %a
  %op2 = load <4 x i64>, ptr %b
  %1 = zext <4 x i64> %op1 to <4 x i128>
  %2 = zext <4 x i64> %op2 to <4 x i128>
  %mul = mul <4 x i128> %1, %2
  %shr = lshr <4 x i128> %mul, splat (i128 64)
  %res = trunc <4 x i128> %shr to <4 x i64>
  store <4 x i64> %res, ptr %a
  ret void
}