; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64 -global-isel -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI

define <4 x i16> @addls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: addls_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    saddl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addls_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = add <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @addws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: addws_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    saddw v0.8h, v0.8h, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addws_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = add <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @addlu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: addlu_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    uaddl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addlu_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = add <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @addwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: addwu_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    uaddw v0.8h, v0.8h, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addwu_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = add <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @subls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: subls_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ssubl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subls_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = sub <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @subws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: subws_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    ssubw v0.8h, v0.8h, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subws_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = sub <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @sublu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: sublu_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    usubl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sublu_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = sub <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @subwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: subwu_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    usubw v0.8h, v0.8h, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subwu_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = sub <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @mulls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: mulls_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulls_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = mul <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @mulws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-LABEL: mulws_v8i8_0:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-NEXT:    ret
entry:
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = mul <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @mullu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: mullu_v8i8_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mullu_v8i8_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = mul <4 x i16> %t0, %t1
  ret <4 x i16> %m
}

define <4 x i16> @mulwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
; CHECK-LABEL: mulwu_v8i8_0:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-NEXT:    ret
entry:
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %m = mul <4 x i16> %s0, %t1
  ret <4 x i16> %m
}

define <2 x i32> @addls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: addls_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    saddl v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addls_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = add <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @addws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: addws_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addws_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = add <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @addlu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: addlu_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    uaddl v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addlu_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = add <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @addwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: addwu_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addwu_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = add <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @subls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: subls_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ssubl v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subls_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = sub <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @subws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: subws_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    ssubw v0.4s, v0.4s, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subws_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = sub <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @sublu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: sublu_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    usubl v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sublu_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = sub <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @subwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: subwu_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    usubw v0.4s, v0.4s, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subwu_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = sub <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @mulls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: mulls_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulls_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = mul <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @mulws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-LABEL: mulws_v4i16_0:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-NEXT:    ret
entry:
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = mul <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @mullu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-SD-LABEL: mullu_v4i16_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    umull v0.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mullu_v4i16_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = mul <2 x i32> %t0, %t1
  ret <2 x i32> %m
}

define <2 x i32> @mulwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
; CHECK-LABEL: mulwu_v4i16_0:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-NEXT:    ret
entry:
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
  %m = mul <2 x i32> %s0, %t1
  ret <2 x i32> %m
}

define <1 x i64> @addls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: addls_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    saddl v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addls_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    add x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <2 x i32> %s0 to <2 x i64>
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = add <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @addws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: addws_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    saddw v0.2d, v0.2d, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addws_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    add x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = add <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @addlu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: addlu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    uaddl v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addlu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    add x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <2 x i32> %s0 to <2 x i64>
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = add <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @addwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: addwu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    uaddw v0.2d, v0.2d, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: addwu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    add x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = add <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @subls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: subls_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ssubl v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subls_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    sub x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <2 x i32> %s0 to <2 x i64>
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = sub <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @subws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: subws_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    ssubw v0.2d, v0.2d, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subws_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    sub x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = sub <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @sublu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: sublu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    usubl v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sublu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    sub x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <2 x i32> %s0 to <2 x i64>
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = sub <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @subwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: subwu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    usubw v0.2d, v0.2d, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: subwu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    sub x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = sub <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @mulls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: mulls_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulls_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = sext <2 x i32> %s0 to <2 x i64>
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = mul <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @mulws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: mulws_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    fmov x8, d0
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    mul x8, x8, x9
; CHECK-SD-NEXT:    fmov d0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulws_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = mul <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @mullu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: mullu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    umull v0.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mullu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    umull x8, w8, w9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s0s = zext <2 x i32> %s0 to <2 x i64>
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = mul <1 x i64> %t0, %t1
  ret <1 x i64> %m
}

define <1 x i64> @mulwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
; CHECK-SD-LABEL: mulwu_v2i32_0:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    fmov x8, d0
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    mul x8, x8, x9
; CHECK-SD-NEXT:    fmov d0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulwu_v2i32_0:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    ret
entry:
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
  %m = mul <1 x i64> %s0, %t1
  ret <1 x i64> %m
}

define <2 x i8> @extract_scalable_vec() vscale_range(1,16) "target-features"="+sve" {
; CHECK-SD-LABEL: extract_scalable_vec:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov x8, xzr
; CHECK-SD-NEXT:    index z1.s, #2, #3
; CHECK-SD-NEXT:    ldr h0, [x8]
; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: extract_scalable_vec:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, xzr
; CHECK-GI-NEXT:    mov w9, #1 // =0x1
; CHECK-GI-NEXT:    ld1 { v0.b }[0], [x8]
; CHECK-GI-NEXT:    ldr b1, [x9]
; CHECK-GI-NEXT:    adrp x8, .LCPI36_0
; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI36_0]
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ret
entry:
  %0 = load <2 x i8>, ptr null, align 2
  %mul = mul <2 x i8> %0, <i8 2, i8 5>
  ret <2 x i8> %mul
}