; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -S | FileCheck %s

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

; tests skipping iterations within a VF through break/continue/gotos.

; The main difficulty in vectorizing these loops in test1,test2 and test3 is
; safely speculating that the widened load of A[i] should not fault if the
; scalarized loop does not fault. For example, the
; original load in the scalar loop may not fault, but the last iteration of the
; vectorized load can fault (if it crosses a page boudary for example).
; This last vector iteration is where *one* of the
; scalar iterations lead to the early exit.

; int test(int *A, int Length) {
;   for (int i = 0; i < Length; i++) {
;     if (A[i] > 10.0) goto end;
;     A[i] = 0;
;   }
; end:
;   return 0;
; }
define i32 @test1(ptr nocapture %A, i32 %Length) {
; CHECK-LABEL: define i32 @test1(
; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LENGTH:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LENGTH]], 0
; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[END:.*]]
; CHECK:       [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_ELSE:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA0:![0-9]+]]
; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 10
; CHECK-NEXT:    br i1 [[CMP1]], label %[[END_LOOPEXIT:.*]], label %[[IF_ELSE]]
; CHECK:       [[IF_ELSE]]:
; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA0]]
; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], [[LENGTH]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[END_LOOPEXIT]]
; CHECK:       [[END_LOOPEXIT]]:
; CHECK-NEXT:    br label %[[END]]
; CHECK:       [[END]]:
; CHECK-NEXT:    ret i32 0
;
entry:
  %cmp8 = icmp sgt i32 %Length, 0
  br i1 %cmp8, label %for.body.preheader, label %end

for.body.preheader:
  br label %for.body

for.body:
  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
  %0 = load i32, ptr %arrayidx, align 4, !tbaa !15
  %cmp1 = icmp sgt i32 %0, 10
  br i1 %cmp1, label %end.loopexit, label %if.else

if.else:
  store i32 0, ptr %arrayidx, align 4, !tbaa !15
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %1 = trunc i64 %indvars.iv.next to i32
  %cmp = icmp slt i32 %1, %Length
  br i1 %cmp, label %for.body, label %end.loopexit

end.loopexit:
  br label %end

end:
  ret i32 0
}

; We don't use anything from within the loop at the early exit path
; so we do not need to know which iteration caused the early exit path.
; bool test2(int *A, int Length, int K) {
;   for (int i = 0; i < Length; i++) {
;     if (A[i] == K) return true;
;   }
;   return false;
; }
; TODO: Today we do not vectorize this, but we could teach the vectorizer, once
; the hard part of proving/speculating A[i:VF - 1] loads does not fault is handled by the
; compiler/hardware.

define i32 @test2(ptr nocapture %A, i32 %Length, i32 %K) {
; CHECK-LABEL: define i32 @test2(
; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LENGTH:%.*]], i32 [[K:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LENGTH]], 0
; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[END:.*]]
; CHECK:       [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_ELSE:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LD]], [[K]]
; CHECK-NEXT:    br i1 [[CMP1]], label %[[END_LOOPEXIT:.*]], label %[[IF_ELSE]]
; CHECK:       [[IF_ELSE]]:
; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TRUNC]], [[LENGTH]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[END_LOOPEXIT]]
; CHECK:       [[END_LOOPEXIT]]:
; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i32 [ 1, %[[FOR_BODY]] ], [ 0, %[[IF_ELSE]] ]
; CHECK-NEXT:    br label %[[END]]
; CHECK:       [[END]]:
; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[RESULT_LCSSA]], %[[END_LOOPEXIT]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    ret i32 [[RESULT]]
;
entry:
  %cmp8 = icmp sgt i32 %Length, 0
  br i1 %cmp8, label %for.body.preheader, label %end

for.body.preheader:
  br label %for.body

for.body:
  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
  %ld = load i32, ptr %arrayidx, align 4
  %cmp1 = icmp eq i32 %ld, %K
  br i1 %cmp1, label %end.loopexit, label %if.else

if.else:
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %trunc = trunc i64 %indvars.iv.next to i32
  %cmp = icmp slt i32 %trunc, %Length
  br i1 %cmp, label %for.body, label %end.loopexit

end.loopexit:
  %result.lcssa = phi i32 [ 1, %for.body ], [ 0, %if.else ]
  br label %end

end:
  %result = phi i32 [ %result.lcssa, %end.loopexit ], [ 0, %entry ]
  ret i32 %result
}

; We use the IV in the early exit
; so we need to know which iteration caused the early exit path.
; int test3(int *A, int Length, int K) {
;   for (int i = 0; i < Length; i++) {
;     if (A[i] == K) return i;
;   }
;   return -1;
; }
; TODO: Today we do not vectorize this, but we could teach the vectorizer (once
; we handle the speculation safety of the widened load).
define i32 @test3(ptr nocapture %A, i32 %Length, i32 %K) {
; CHECK-LABEL: define i32 @test3(
; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LENGTH:%.*]], i32 [[K:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LENGTH]], 0
; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[END:.*]]
; CHECK:       [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[IF_ELSE:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LD]], [[K]]
; CHECK-NEXT:    br i1 [[CMP1]], label %[[END_LOOPEXIT:.*]], label %[[IF_ELSE]]
; CHECK:       [[IF_ELSE]]:
; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TRUNC]], [[LENGTH]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[END_LOOPEXIT]]
; CHECK:       [[END_LOOPEXIT]]:
; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i64 [ [[INDVARS_IV]], %[[FOR_BODY]] ], [ -1, %[[IF_ELSE]] ]
; CHECK-NEXT:    [[RES_TRUNC:%.*]] = trunc i64 [[RESULT_LCSSA]] to i32
; CHECK-NEXT:    br label %[[END]]
; CHECK:       [[END]]:
; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[RES_TRUNC]], %[[END_LOOPEXIT]] ], [ -1, %[[ENTRY]] ]
; CHECK-NEXT:    ret i32 [[RESULT]]
;
entry:
  %cmp8 = icmp sgt i32 %Length, 0
  br i1 %cmp8, label %for.body.preheader, label %end

for.body.preheader:
  br label %for.body

for.body:
  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
  %ld = load i32, ptr %arrayidx, align 4
  %cmp1 = icmp eq i32 %ld, %K
  br i1 %cmp1, label %end.loopexit, label %if.else

if.else:
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %trunc = trunc i64 %indvars.iv.next to i32
  %cmp = icmp slt i32 %trunc, %Length
  br i1 %cmp, label %for.body, label %end.loopexit

end.loopexit:
  %result.lcssa = phi i64 [ %indvars.iv, %for.body ], [ -1, %if.else ]
  %res.trunc = trunc i64 %result.lcssa to i32
  br label %end

end:
  %result = phi i32 [ %res.trunc, %end.loopexit ], [ -1, %entry ]
  ret i32 %result
}

; bool test4(int *A, int Length, int K, int J) {
;   for (int i = 0; i < Length; i++) {
;     if (A[i] == K) continue;
;     A[i] = J;
;   }
; }
; For this test, we vectorize and generate predicated stores to A[i].
define void @test4(ptr nocapture %A, i32 %Length, i32 %K, i32 %J) {
; CHECK-LABEL: define void @test4(
; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LENGTH:%.*]], i32 [[K:%.*]], i32 [[J:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[LENGTH]], 0
; CHECK-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[END_LOOPEXIT:.*]]
; CHECK:       [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[LENGTH]] to i64
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK:       [[PRED_STORE_IF]]:
; CHECK-NEXT:    store i32 [[J]], ptr [[TMP1]], align 4
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; CHECK:       [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; CHECK:       [[PRED_STORE_IF1]]:
; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
; CHECK-NEXT:    store i32 [[J]], ptr [[TMP6]], align 4
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; CHECK:       [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK:       [[PRED_STORE_IF3]]:
; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
; CHECK-NEXT:    store i32 [[J]], ptr [[TMP9]], align 4
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
; CHECK:       [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; CHECK:       [[PRED_STORE_IF5]]:
; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
; CHECK-NEXT:    store i32 [[J]], ptr [[TMP12]], align 4
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
; CHECK:       [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], label %[[END_LOOPEXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LD]], [[K]]
; CHECK-NEXT:    br i1 [[CMP1]], label %[[LATCH]], label %[[IF_ELSE:.*]]
; CHECK:       [[IF_ELSE]]:
; CHECK-NEXT:    store i32 [[J]], ptr [[ARRAYIDX]], align 4
; CHECK-NEXT:    br label %[[LATCH]]
; CHECK:       [[LATCH]]:
; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TRUNC]], [[LENGTH]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY]], label %[[END_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[END_LOOPEXIT_LOOPEXIT]]:
; CHECK-NEXT:    br label %[[END_LOOPEXIT]]
; CHECK:       [[END_LOOPEXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %cmp8 = icmp sgt i32 %Length, 0
  br i1 %cmp8, label %for.body.preheader, label %end.loopexit

for.body.preheader:
  br label %for.body

for.body:
  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %for.body.preheader ]
  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %trunc = trunc i64 %indvars.iv.next to i32
  %ld = load i32, ptr %arrayidx, align 4
  %cmp1 = icmp eq i32 %ld, %K
  br i1 %cmp1, label %latch, label %if.else

if.else:
  store i32 %J, ptr %arrayidx, align 4
  br label %latch

latch:
  %cmp = icmp slt i32 %trunc, %Length
  br i1 %cmp, label %for.body, label %end.loopexit

end.loopexit:
  ret void
}
!15 = !{!16, !16, i64 0}
!16 = !{!"int", !17, i64 0}
!17 = !{!"omnipotent char", !18, i64 0}
!18 = !{!"Simple C/C++ TBAA"}