; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6

; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s

define i32 @vec_mul_reduce_add(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i32 %N) {
; CHECK-LABEL: define i32 @vec_mul_reduce_add(
; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[CMP8:%.*]] = icmp eq i32 [[N]], 0
; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], 3
; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 2
; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], -4
; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 2
; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i32 [[TMP4]], 1
; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP5]])
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[LSR_IV2:%.*]] = phi ptr [ [[SCEVGEP3:%.*]], %[[VECTOR_BODY]] ], [ [[A]], %[[VECTOR_PH]] ]
; CHECK-NEXT:    [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[B]], %[[VECTOR_PH]] ]
; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP7]])
; CHECK-NEXT:    [[TMP9]] = sub i32 [[TMP7]], 4
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV2]], <4 x i1> [[TMP8]], <4 x i32> undef)
; CHECK-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV]], <4 x i1> [[TMP8]], <4 x i32> undef)
; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD13]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT:    [[TMP11]] = add nsw <4 x i32> [[TMP10]], [[VEC_PHI]]
; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i32, ptr [[LSR_IV]], i32 4
; CHECK-NEXT:    [[SCEVGEP3]] = getelementptr i32, ptr [[LSR_IV2]], i32 4
; CHECK-NEXT:    [[TMP12]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP6]], i32 1)
; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i32 [[TMP12]], 0
; CHECK-NEXT:    br i1 [[TMP13]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[TMP11]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP]]
; CHECK:       [[FOR_COND_CLEANUP]]:
; CHECK-NEXT:    [[RES_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT:    ret i32 [[RES_0_LCSSA]]
;
entry:
  %cmp8 = icmp eq i32 %N, 0
  %0 = add i32 %N, 3
  %1 = lshr i32 %0, 2
  %2 = shl nuw i32 %1, 2
  %3 = add i32 %2, -4
  %4 = lshr i32 %3, 2
  %5 = add nuw nsw i32 %4, 1
  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %start = call i32 @llvm.start.loop.iterations.i32(i32 %5)
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %lsr.iv2 = phi ptr [ %scevgep3, %vector.body ], [ %a, %vector.ph ]
  %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %b, %vector.ph ]
  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %9, %vector.body ]
  %6 = phi i32 [ %start, %vector.ph ], [ %10, %vector.body ]
  %7 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv2, i32 4, <4 x i1> %7, <4 x i32> undef)
  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv, i32 4, <4 x i1> %7, <4 x i32> undef)
  %8 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
  %9 = add nsw <4 x i32> %8, %vec.phi
  %index.next = add i32 %index, 4
  %scevgep = getelementptr i32, ptr %lsr.iv, i32 4
  %scevgep3 = getelementptr i32, ptr %lsr.iv2, i32 4
  %10 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %6, i32 1)
  %11 = icmp ne i32 %10, 0
  br i1 %11, label %vector.body, label %middle.block

middle.block:                                     ; preds = %vector.body
  %12 = select <4 x i1> %7, <4 x i32> %9, <4 x i32> %vec.phi
  %13 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %12)
  br label %for.cond.cleanup

for.cond.cleanup:                                 ; preds = %middle.block, %entry
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %13, %middle.block ]
  ret i32 %res.0.lcssa
}

declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.start.loop.iterations.i32(i32)
declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)