; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s

define <16 x i1> @whilewr_8(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 1)
  ret <16 x i1> %0
}

define <8 x i1> @whilewr_16(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 2)
  ret <8 x i1> %0
}

define <4 x i1> @whilewr_32(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 4)
  ret <4 x i1> %0
}

define <2 x i1> @whilewr_64(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.d, x0, x1
; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.2s, v0.2d
; CHECK-NEXT:    ret
entry:
  %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 8)
  ret <2 x i1> %0
}

define <16 x i1> @whilerw_8(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i64(i64 %a, i64 %b, i64 1)
  ret <16 x i1> %0
}

define <8 x i1> @whilerw_16(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1.i64(i64 %a, i64 %b, i64 2)
  ret <8 x i1> %0
}

define <4 x i1> @whilerw_32(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1.i64(i64 %a, i64 %b, i64 4)
  ret <4 x i1> %0
}

define <2 x i1> @whilerw_64(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilerw p0.d, x0, x1
; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.2s, v0.2d
; CHECK-NEXT:    ret
entry:
  %0 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 8)
  ret <2 x i1> %0
}

define <32 x i1> @whilewr_8_split(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_8_split:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    mov w10, #16 // =0x10
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    csinv x9, x9, xzr, ge
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    whilelo p1.b, x10, x9
; CHECK-NEXT:    adrp x9, .LCPI8_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI8_0]
; CHECK-NEXT:    mov z1.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    str h0, [x8]
; CHECK-NEXT:    str h1, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 1)
  ret <32 x i1> %0
}

define <64 x i1> @whilewr_8_split2(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_8_split2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    mov w10, #48 // =0x30
; CHECK-NEXT:    mov w11, #16 // =0x10
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    mov w12, #32 // =0x20
; CHECK-NEXT:    csinv x9, x9, xzr, ge
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    whilelo p1.b, x10, x9
; CHECK-NEXT:    adrp x10, .LCPI9_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    whilelo p0.b, x12, x9
; CHECK-NEXT:    ldr q1, [x10, :lo12:.LCPI9_0]
; CHECK-NEXT:    mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    whilelo p1.b, x11, x9
; CHECK-NEXT:    mov z3.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    mov z4.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v1.16b
; CHECK-NEXT:    and v3.16b, v3.16b, v1.16b
; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v2.16b, v2.16b, v2.16b
; CHECK-NEXT:    addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v2.16b, v2.16b, v2.16b
; CHECK-NEXT:    addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v2.16b, v2.16b, v2.16b
; CHECK-NEXT:    addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    str h0, [x8]
; CHECK-NEXT:    str h2, [x8, #6]
; CHECK-NEXT:    str h3, [x8, #4]
; CHECK-NEXT:    str h1, [x8, #2]
; CHECK-NEXT:    ret
entry:
  %0 = call <64 x i1> @llvm.loop.dependence.war.mask.v64i1.i64(i64 %a, i64 %b, i64 1)
  ret <64 x i1> %0
}

define <16 x i1> @whilewr_16_expand(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_16_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub x8, x1, x0
; CHECK-NEXT:    add x8, x8, x8, lsr #63
; CHECK-NEXT:    asr x8, x8, #1
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 2)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_16_expand2(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_16_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    mov w10, #16 // =0x10
; CHECK-NEXT:    add x9, x9, x9, lsr #63
; CHECK-NEXT:    asr x9, x9, #1
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    csinv x9, x9, xzr, ge
; CHECK-NEXT:    whilelo p0.b, x10, x9
; CHECK-NEXT:    whilelo p1.b, xzr, x9
; CHECK-NEXT:    adrp x9, .LCPI11_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI11_0]
; CHECK-NEXT:    mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 2)
  ret <32 x i1> %0
}

define <8 x i1> @whilewr_32_expand(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #3
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #2
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
  ret <8 x i1> %0
}

define <16 x i1> @whilewr_32_expand2(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #3
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #2
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 4)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_32_expand3(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32_expand3:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x9, x1, x0
; CHECK-NEXT:    add x10, x9, #3
; CHECK-NEXT:    csel x9, x10, x9, mi
; CHECK-NEXT:    mov w10, #16 // =0x10
; CHECK-NEXT:    asr x9, x9, #2
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    csinv x9, x9, xzr, ge
; CHECK-NEXT:    whilelo p0.b, x10, x9
; CHECK-NEXT:    whilelo p1.b, xzr, x9
; CHECK-NEXT:    adrp x9, .LCPI14_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI14_0]
; CHECK-NEXT:    mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 4)
  ret <32 x i1> %0
}

define <4 x i1> @whilewr_64_expand(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64_expand:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.h, xzr, x8
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 8)
  ret <4 x i1> %0
}

define <8 x i1> @whilewr_64_expand2(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64_expand2:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 8)
  ret <8 x i1> %0
}

define <16 x i1> @whilewr_64_expand3(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64_expand3:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x8, x1, x0
; CHECK-NEXT:    add x9, x8, #7
; CHECK-NEXT:    csel x8, x9, x8, mi
; CHECK-NEXT:    asr x8, x8, #3
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
  ret <16 x i1> %0
}

define <32 x i1> @whilewr_64_expand4(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64_expand4:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    subs x9, x1, x0
; CHECK-NEXT:    add x10, x9, #7
; CHECK-NEXT:    csel x9, x10, x9, mi
; CHECK-NEXT:    mov w10, #16 // =0x10
; CHECK-NEXT:    asr x9, x9, #3
; CHECK-NEXT:    cmp x9, #1
; CHECK-NEXT:    csinv x9, x9, xzr, ge
; CHECK-NEXT:    whilelo p0.b, x10, x9
; CHECK-NEXT:    whilelo p1.b, xzr, x9
; CHECK-NEXT:    adrp x9, .LCPI18_0
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI18_0]
; CHECK-NEXT:    mov z2.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    str h0, [x8, #2]
; CHECK-NEXT:    str h1, [x8]
; CHECK-NEXT:    ret
entry:
  %0 = call <32 x i1> @llvm.loop.dependence.war.mask.v32i1.i64(i64 %a, i64 %b, i64 8)
  ret <32 x i1> %0
}

define <9 x i1> @whilewr_8_widen(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_8_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    umov w9, v0.b[0]
; CHECK-NEXT:    umov w10, v0.b[1]
; CHECK-NEXT:    umov w11, v0.b[2]
; CHECK-NEXT:    umov w12, v0.b[7]
; CHECK-NEXT:    and w9, w9, #0x1
; CHECK-NEXT:    bfi w9, w10, #1, #1
; CHECK-NEXT:    umov w10, v0.b[3]
; CHECK-NEXT:    bfi w9, w11, #2, #1
; CHECK-NEXT:    umov w11, v0.b[4]
; CHECK-NEXT:    bfi w9, w10, #3, #1
; CHECK-NEXT:    umov w10, v0.b[5]
; CHECK-NEXT:    bfi w9, w11, #4, #1
; CHECK-NEXT:    umov w11, v0.b[6]
; CHECK-NEXT:    bfi w9, w10, #5, #1
; CHECK-NEXT:    umov w10, v0.b[8]
; CHECK-NEXT:    bfi w9, w11, #6, #1
; CHECK-NEXT:    ubfiz w11, w12, #7, #1
; CHECK-NEXT:    orr w9, w9, w11
; CHECK-NEXT:    orr w9, w9, w10, lsl #8
; CHECK-NEXT:    and w9, w9, #0x1ff
; CHECK-NEXT:    strh w9, [x8]
; CHECK-NEXT:    ret
entry:
  %0 = call <9 x i1> @llvm.loop.dependence.war.mask.v9i1.i64(i64 %a, i64 %b, i64 1)
  ret <9 x i1> %0
}

define <7 x i1> @whilewr_16_widen(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_16_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.h, x0, x1
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.8b, v0.8h
; CHECK-NEXT:    umov w0, v0.b[0]
; CHECK-NEXT:    umov w1, v0.b[1]
; CHECK-NEXT:    umov w2, v0.b[2]
; CHECK-NEXT:    umov w3, v0.b[3]
; CHECK-NEXT:    umov w4, v0.b[4]
; CHECK-NEXT:    umov w5, v0.b[5]
; CHECK-NEXT:    umov w6, v0.b[6]
; CHECK-NEXT:    ret
entry:
  %0 = call <7 x i1> @llvm.loop.dependence.war.mask.v7i1.i64(i64 %a, i64 %b, i64 2)
  ret <7 x i1> %0
}

define <3 x i1> @whilewr_32_widen(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32_widen:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    umov w0, v0.h[0]
; CHECK-NEXT:    umov w1, v0.h[1]
; CHECK-NEXT:    umov w2, v0.h[2]
; CHECK-NEXT:    ret
entry:
  %0 = call <3 x i1> @llvm.loop.dependence.war.mask.v3i1.i64(i64 %a, i64 %b, i64 4)
  ret <3 x i1> %0
}

define <16 x i1> @whilewr_badimm(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_badimm:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov x8, #6148914691236517205 // =0x5555555555555555
; CHECK-NEXT:    sub x9, x1, x0
; CHECK-NEXT:    movk x8, #21846
; CHECK-NEXT:    smulh x8, x9, x8
; CHECK-NEXT:    add x8, x8, x8, lsr #63
; CHECK-NEXT:    cmp x8, #1
; CHECK-NEXT:    csinv x8, x8, xzr, ge
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 3)
  ret <16 x i1> %0
}

; Scalarizing <1 x i1> types

define <1 x i1> @whilewr_8_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_8_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 1)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_16_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_16_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 2)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_32_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_32_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 4)
  ret <1 x i1> %0
}

define <1 x i1> @whilewr_64_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_64_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1.i64(i64 %a, i64 %b, i64 8)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_8_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_8_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 1)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_16_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_16_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 2)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_32_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_32_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 4)
  ret <1 x i1> %0
}

define <1 x i1> @whilerw_64_scalarize(i64 %a, i64 %b) {
; CHECK-LABEL: whilerw_64_scalarize:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w0, #1 // =0x1
; CHECK-NEXT:    ret
entry:
  %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1.i64(i64 %a, i64 %b, i64 8)
  ret <1 x i1> %0
}

define <8 x i1> @whilewr_extract_v8i1(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_extract_v8i1:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 1)
  ret <8 x i1> %0
}

define <4 x i1> @whilewr_extract_v4i1(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_extract_v4i1:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.b, x0, x1
; CHECK-NEXT:    punpklo p0.h, p0.b
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 1)
  ret <4 x i1> %0
}

define <2 x i1> @whilewr_extract_v2i1(i64 %a, i64 %b) {
; CHECK-LABEL: whilewr_extract_v2i1:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    whilewr p0.s, x0, x1
; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 4)
  ret <2 x i1> %0
}

define <16 x i1> @war_mask_i16(i16 %a, i16 %b) {
; CHECK-LABEL: war_mask_i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    and w8, w0, #0xffff
; CHECK-NEXT:    and w9, w1, #0xffff
; CHECK-NEXT:    whilewr p0.b, x8, x9
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i16(i16 %a, i16 %b, i16 1)
  ret <16 x i1> %0
}

define <16 x i1> @raw_mask_i16(i16 %a, i16 %b) {
; CHECK-LABEL: raw_mask_i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    and w8, w0, #0xffff
; CHECK-NEXT:    and w9, w1, #0xffff
; CHECK-NEXT:    whilerw p0.b, x8, x9
; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
entry:
  %0 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i16(i16 %a, i16 %b, i16 1)
  ret <16 x i1> %0
}