; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sme-i16i64 -mattr=+sme-f64f64 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s

;
; ADD Multi-Single x2
;

define void @multi_vector_add_write_single_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,  <vscale x 4 x i32> %zm) {
; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }, z2.s
; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }, z2.s
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32 %slice,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zm)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32 %slice.7,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zm)
  ret void
}

define void @multi_vector_add_write_single_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,  <vscale x 2 x i64> %zm) {
; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }, z2.d
; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }, z2.d
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32 %slice,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zm)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32 %slice.7,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zm)
  ret void
}

;
; ADD Multi-Single x4
;

define void @multi_vector_add_write_single_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }, z4.s
; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }, z4.s
; CHECK-NEXT:    ret
                                               <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                               <vscale x 4 x i32> %zm) {
  call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32 %slice,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                                       <vscale x 4 x i32> %zm)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32 %slice.7,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                                       <vscale x 4 x i32> %zm)
  ret void
}

define void @multi_vector_add_write_single_za_vg1x4_i64(i32 %slice,
; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }, z4.d
; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }, z4.d
; CHECK-NEXT:    ret
                                               <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                               <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                               <vscale x 2 x i64> %zm) {
  call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32 %slice,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                                       <vscale x 2 x i64> %zm)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32 %slice.7,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                                       <vscale x 2 x i64> %zm)
  ret void
}

;
; ADD Multi-Multi x2
;

define void @multi_vector_add_write_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }, { z2.s, z3.s }
; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }, { z2.s, z3.s }
; CHECK-NEXT:    ret
                                        <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2) {
  call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32 %slice,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32 %slice.7,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                       <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2)
  ret void
}


define void @multi_vector_add_write_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }, { z2.d, z3.d }
; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }, { z2.d, z3.d }
; CHECK-NEXT:    ret
                                        <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2) {
  call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32 %slice,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32 %slice.7,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2)
  ret void
}


;
; ADD Multi-Multi x4
;

define void @multi_vector_add_write_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }, { z4.s - z7.s }
; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }, { z4.s - z7.s }
; CHECK-NEXT:    ret
                                        <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                        <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,
                                        <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3) {
  call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32 %slice,
                                                      <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                      <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                                      <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,
                                                      <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32 %slice.7,
                                                       <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                                                      <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,
                                                       <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,
                                                      <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3)
  ret void
}

define void @multi_vector_add_write_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }, { z4.d - z7.d }
; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }, { z4.d - z7.d }
; CHECK-NEXT:    ret
                                        <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                        <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,
                                        <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3) {
  call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32 %slice,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                                       <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,
                                                       <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32 %slice.7,
                                                       <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                                                       <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,
                                                       <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,
                                                       <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3)
  ret void
}

;
; ADD and accumulate into ZA
;
; x2
define void @multi_vector_add_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1) {
; CHECK-LABEL: multi_vector_add_za_vg1x2_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx2], { z0.s, z1.s }
; CHECK-NEXT:    add za.s[w8, 7, vgx2], { z0.s, z1.s }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32 %slice,<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32 %slice.7, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)
  ret void
}

define void @multi_vector_add_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1) {
; CHECK-LABEL: multi_vector_add_za_vg1x2_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx2], { z0.d, z1.d }
; CHECK-NEXT:    add za.d[w8, 7, vgx2], { z0.d, z1.d }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32 %slice.7, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1)
  ret void
}

define void @multi_vector_add_za_vg1x2_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1) {
; CHECK-LABEL: multi_vector_add_za_vg1x2_f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    fadd za.s[w8, 0, vgx2], { z0.s, z1.s }
; CHECK-NEXT:    fadd za.s[w8, 7, vgx2], { z0.s, z1.s }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32 %slice,
                                                       <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32 %slice.7,
                                                       <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1)
  ret void
}

define void @multi_vector_add_za_vg1x2_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1) {
; CHECK-LABEL: multi_vector_add_za_vg1x2_f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z0.d, z1.d }
; CHECK-NEXT:    fadd za.d[w8, 7, vgx2], { z0.d, z1.d }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 %slice,
                                                       <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 %slice.7,
                                                       <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1)
  ret void
}

define void @multi_vector_add_za_vg1x2_f64_tuple(i64 %stride, ptr %ptr) {
; CHECK-LABEL: multi_vector_add_za_vg1x2_f64_tuple:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ptrue pn8.b
; CHECK-NEXT:    add x9, x1, x0
; CHECK-NEXT:    mov w8, wzr
; CHECK-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x1]
; CHECK-NEXT:    ld1d { z17.d, z25.d }, pn8/z, [x9]
; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z16.d, z17.d }
; CHECK-NEXT:    fadd za.d[w8, 0, vgx2], { z24.d, z25.d }
; CHECK-NEXT:    ret
entry:
  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
  %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)
  %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 0
  %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 1
  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
  %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)
  %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0
  %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %5)
  call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %6)
  ret void
}

; x4

define void @multi_vector_add_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3) {
; CHECK-LABEL: multi_vector_add_za_vg1x4_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.s[w8, 0, vgx4], { z0.s - z3.s }
; CHECK-NEXT:    add za.s[w8, 7, vgx4], { z0.s - z3.s }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32 %slice,
                  <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                  <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32 %slice.7,
                   <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,
                   <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3)
  ret void
}

define void @multi_vector_add_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3) {
; CHECK-LABEL: multi_vector_add_za_vg1x4_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add za.d[w8, 0, vgx4], { z0.d - z3.d }
; CHECK-NEXT:    add za.d[w8, 7, vgx4], { z0.d - z3.d }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32 %slice,
                  <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                  <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32 %slice.7,
                  <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,
                  <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3)
  ret void
}

define void @multi_vector_add_za_vg1x4_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3) {
; CHECK-LABEL: multi_vector_add_za_vg1x4_f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z0.s - z3.s }
; CHECK-NEXT:    fadd za.s[w8, 7, vgx4], { z0.s - z3.s }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 %slice,
                  <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,
                  <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 %slice.7,
                  <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,
                  <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3)
  ret void
}

define void @multi_vector_add_za_vg1x4_f32_tuple(i64 %stride, ptr %ptr) {
; CHECK-LABEL: multi_vector_add_za_vg1x4_f32_tuple:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    lsl x9, x0, #1
; CHECK-NEXT:    add x10, x1, x0
; CHECK-NEXT:    ptrue pn8.b
; CHECK-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]
; CHECK-NEXT:    ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x10]
; CHECK-NEXT:    mov w8, wzr
; CHECK-NEXT:    add x11, x1, x9
; CHECK-NEXT:    add x9, x10, x9
; CHECK-NEXT:    ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x11]
; CHECK-NEXT:    ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x9]
; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z16.s - z19.s }
; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z20.s - z23.s }
; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z24.s - z27.s }
; CHECK-NEXT:    fadd za.s[w8, 0, vgx4], { z28.s - z31.s }
; CHECK-NEXT:    ret
entry:
  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
  %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)
  %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 0
  %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 1
  %4 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 2
  %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 3
  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride
  %6 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)
  %7 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 0
  %8 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 1
  %9 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 2
  %10 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 3
  %mul3 = shl i64 %stride, 1
  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3
  %11 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx4)
  %12 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 0
  %13 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 1
  %14 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 2
  %15 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 3
  %mul5 = mul i64 %stride, 3
  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5
  %16 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx6)
  %17 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 0
  %18 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 1
  %19 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 2
  %20 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 3
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %2, <vscale x 4 x float> %7, <vscale x 4 x float> %12, <vscale x 4 x float> %17)
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %3, <vscale x 4 x float> %8, <vscale x 4 x float> %13, <vscale x 4 x float> %18)
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %4, <vscale x 4 x float> %9, <vscale x 4 x float> %14, <vscale x 4 x float> %19)
  call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %5, <vscale x 4 x float> %10, <vscale x 4 x float> %15, <vscale x 4 x float> %20)
  ret void
}

define void @multi_vector_add_za_vg1x4_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3) {
; CHECK-LABEL: multi_vector_add_za_vg1x4_f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    fadd za.d[w8, 0, vgx4], { z0.d - z3.d }
; CHECK-NEXT:    fadd za.d[w8, 7, vgx4], { z0.d - z3.d }
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32 %slice,
                  <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1,
                  <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3)
  %slice.7 = add i32 %slice, 7
  call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32 %slice.7,
                  <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1,
                  <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3)
  ret void
}

;
; ADD Vectors Multi-Single x2
;

define { <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vec_add_single_x2_s8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2, <vscale x 16 x i8> %zm) {
; CHECK-LABEL: multi_vec_add_single_x2_s8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z5.d, z2.d
; CHECK-NEXT:    mov z4.d, z1.d
; CHECK-NEXT:    add { z4.b, z5.b }, { z4.b, z5.b }, z3.b
; CHECK-NEXT:    mov z0.d, z4.d
; CHECK-NEXT:    mov z1.d, z5.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> }
               @llvm.aarch64.sve.add.single.x2.nxv16i8(<vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2,
                                                         <vscale x 16 x i8> %zm)
  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}

define { <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vec_add_single_x2_s16(<vscale x 8 x i16> %unused, <vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2, <vscale x 8 x i16> %zm) {
; CHECK-LABEL: multi_vec_add_single_x2_s16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z5.d, z2.d
; CHECK-NEXT:    mov z4.d, z1.d
; CHECK-NEXT:    add { z4.h, z5.h }, { z4.h, z5.h }, z3.h
; CHECK-NEXT:    mov z0.d, z4.d
; CHECK-NEXT:    mov z1.d, z5.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> }
               @llvm.aarch64.sve.add.single.x2.nxv8i16(<vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2,
                                                         <vscale x 8 x i16> %zm)
  ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res
}

define { <vscale x 4 x i32>, <vscale x 4 x i32> } @multi_vec_add_single_x2_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2, <vscale x 4 x i32> %zm) {
; CHECK-LABEL: multi_vec_add_single_x2_s32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z5.d, z2.d
; CHECK-NEXT:    mov z4.d, z1.d
; CHECK-NEXT:    add { z4.s, z5.s }, { z4.s, z5.s }, z3.s
; CHECK-NEXT:    mov z0.d, z4.d
; CHECK-NEXT:    mov z1.d, z5.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> }
               @llvm.aarch64.sve.add.single.x2.nxv4i32(<vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2,
                                                         <vscale x 4 x i32> %zm)
  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
}

define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_add_single_x2_s64(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2, <vscale x 2 x i64> %zm) {
; CHECK-LABEL: multi_vec_add_single_x2_s64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z5.d, z2.d
; CHECK-NEXT:    mov z4.d, z1.d
; CHECK-NEXT:    add { z4.d, z5.d }, { z4.d, z5.d }, z3.d
; CHECK-NEXT:    mov z0.d, z4.d
; CHECK-NEXT:    mov z1.d, z5.d
; CHECK-NEXT:    ret
 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> }
              @llvm.aarch64.sve.add.single.x2.nxv2i64(<vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2,
                                                        <vscale x 2 x i64> %zm)
  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
}

;
; ADD Vectors Multi-Single x4
;

define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vec_add_single_x4_s8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2, <vscale x 16 x i8> %zdn3, <vscale x 16 x i8> %zdn4, <vscale x 16 x i8>%zm) {
; CHECK-LABEL: multi_vec_add_single_x4_s8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z27.d, z4.d
; CHECK-NEXT:    mov z26.d, z3.d
; CHECK-NEXT:    mov z25.d, z2.d
; CHECK-NEXT:    mov z24.d, z1.d
; CHECK-NEXT:    add { z24.b - z27.b }, { z24.b - z27.b }, z5.b
; CHECK-NEXT:    mov z0.d, z24.d
; CHECK-NEXT:    mov z1.d, z25.d
; CHECK-NEXT:    mov z2.d, z26.d
; CHECK-NEXT:    mov z3.d, z27.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> }
              @llvm.aarch64.sve.add.single.x4.nxv16i8(<vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2,
                                                        <vscale x 16 x i8> %zdn3, <vscale x 16 x i8> %zdn4,
                                                        <vscale x 16 x i8> %zm)
  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}

define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vec_add_x4_single_s16(<vscale x 8 x i16> %unused, <vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2, <vscale x 8 x i16> %zdn3, <vscale x 8 x i16> %zdn4, <vscale x 8 x i16> %zm) {
; CHECK-LABEL: multi_vec_add_x4_single_s16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z27.d, z4.d
; CHECK-NEXT:    mov z26.d, z3.d
; CHECK-NEXT:    mov z25.d, z2.d
; CHECK-NEXT:    mov z24.d, z1.d
; CHECK-NEXT:    add { z24.h - z27.h }, { z24.h - z27.h }, z5.h
; CHECK-NEXT:    mov z0.d, z24.d
; CHECK-NEXT:    mov z1.d, z25.d
; CHECK-NEXT:    mov z2.d, z26.d
; CHECK-NEXT:    mov z3.d, z27.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> }
              @llvm.aarch64.sve.add.single.x4.nxv8i16(<vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2,
                                                        <vscale x 8 x i16> %zdn3, <vscale x 8 x i16> %zdn4,
                                                        <vscale x 8 x i16> %zm)
  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res
}

define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @multi_vec_add_x4_single_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2, <vscale x 4 x i32> %zdn3, <vscale x 4 x i32> %zdn4, <vscale x 4 x i32> %zm) {
; CHECK-LABEL: multi_vec_add_x4_single_s32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z27.d, z4.d
; CHECK-NEXT:    mov z26.d, z3.d
; CHECK-NEXT:    mov z25.d, z2.d
; CHECK-NEXT:    mov z24.d, z1.d
; CHECK-NEXT:    add { z24.s - z27.s }, { z24.s - z27.s }, z5.s
; CHECK-NEXT:    mov z0.d, z24.d
; CHECK-NEXT:    mov z1.d, z25.d
; CHECK-NEXT:    mov z2.d, z26.d
; CHECK-NEXT:    mov z3.d, z27.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }
              @llvm.aarch64.sve.add.single.x4.nxv4i32(<vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2,
                                                        <vscale x 4 x i32> %zdn3, <vscale x 4 x i32> %zdn4,
                                                        <vscale x 4 x i32> %zm)
  ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res
}

define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_add_x4_single_s64(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2, <vscale x 2 x i64> %zdn3, <vscale x 2 x i64> %zdn4, <vscale x 2 x i64> %zm) {
; CHECK-LABEL: multi_vec_add_x4_single_s64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z27.d, z4.d
; CHECK-NEXT:    mov z26.d, z3.d
; CHECK-NEXT:    mov z25.d, z2.d
; CHECK-NEXT:    mov z24.d, z1.d
; CHECK-NEXT:    add { z24.d - z27.d }, { z24.d - z27.d }, z5.d
; CHECK-NEXT:    mov z0.d, z24.d
; CHECK-NEXT:    mov z1.d, z25.d
; CHECK-NEXT:    mov z2.d, z26.d
; CHECK-NEXT:    mov z3.d, z27.d
; CHECK-NEXT:    ret
  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> }
              @llvm.aarch64.sve.add.single.x4.nxv2i64(<vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2,
                                                        <vscale x 2 x i64> %zdn3, <vscale x 2 x i64> %zdn4,
                                                        <vscale x 2 x i64> %zm)
  ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
}
declare void@llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32, <vscale x 4 x i32>,<vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32, <vscale x 2 x i64>,<vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32, <vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)
declare void@llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32, <vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>)
declare void@llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>)
declare void@llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>)
declare void@llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>)
declare void@llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>)
declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.add.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.add.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.add.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.add.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.add.single.x4.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.add.single.x4.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.add.single.x4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.add.single.x4.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)