; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -disable-loop-idiom-vectorize-find-first-byte -S < %s | FileCheck -check-prefix=DISABLE %s

; Base case based on `libcxx/include/__algorithm/find_first_of.h':
;   char* find_first_of(char *first, char *last, char *s_first, char *s_last) {
;     for (; first != last; ++first)
;       for (char *it = s_first; it != s_last; ++it)
;         if (*first == *it)
;           return first;
;     return last;
;   }
define ptr @find_first_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; CHECK:       [[HEADER_PREHEADER]]:
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoaddr ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoaddr ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoaddr ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sub i64 [[TMP2]], [[TMP3]]
; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0:![0-9]+]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i8, ptr [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 16 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 16
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP6]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 16
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
; CHECK:       [[EXIT_LOOPEXIT]]:
; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT_LOOPEXIT1]]:
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT:    ret ptr [[RES]]
;
; DISABLE-LABEL: define ptr @find_first_of_i8(
; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; DISABLE:       [[HEADER_PREHEADER]]:
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
; DISABLE:       [[EXIT_LOOPEXIT]]:
; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT_LOOPEXIT1]]:
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT]]:
; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; DISABLE-NEXT:    ret ptr [[RES]]
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  ret ptr %res
}

; Equivalent to @find_first_of_i8 but with i16.
; This is accepted and generates a similar loop.
define ptr @find_first_of_i16(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i16(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; CHECK:       [[HEADER_PREHEADER]]:
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoaddr ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sdiv exact i64 [[TMP2]], 2
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoaddr ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoaddr ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sdiv exact i64 [[TMP5]], 2
; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP6]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i16, ptr [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 1 [[PSEARCH]], <vscale x 8 x i1> [[SEARCH_MASKED]], <vscale x 8 x i16> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 8 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP6]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i16, ptr [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 1 [[PNEEDLE]], <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[NEEDLE0_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], <vscale x 8 x i16> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> [[SEARCH_LOAD_VEC]], <8 x i16> [[NEEDLE_VEC]], <vscale x 8 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 8 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 8
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 8 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP8]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 8 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i16, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 8
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP9]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
; CHECK:       [[EXIT_LOOPEXIT]]:
; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT_LOOPEXIT1]]:
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT:    ret ptr [[RES]]
;
; DISABLE-LABEL: define ptr @find_first_of_i16(
; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; DISABLE:       [[HEADER_PREHEADER]]:
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
; DISABLE:       [[EXIT_LOOPEXIT]]:
; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT_LOOPEXIT1]]:
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT]]:
; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; DISABLE-NEXT:    ret ptr [[RES]]
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i16, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i16, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i16, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i16 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i16, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  ret ptr %res
}

; Same as @find_first_of_i8 but with two intermediate exit blocks for the
; "success" (exit_succ) and "failure" (exit_fail) paths.
define ptr @find_first_of_i8_multi_exit(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]]
; CHECK:       [[HEADER_PREHEADER]]:
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoaddr ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoaddr ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoaddr ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sub i64 [[TMP2]], [[TMP3]]
; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i8, ptr [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 16 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 16
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP6]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[EXIT_SUCC:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 16
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_FAIL_LOOPEXIT:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_SUCC]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT]], label %[[HEADER]]
; CHECK:       [[EXIT_SUCC]]:
; CHECK-NEXT:    [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT_FAIL_LOOPEXIT]]:
; CHECK-NEXT:    br label %[[EXIT_FAIL]]
; CHECK:       [[EXIT_FAIL]]:
; CHECK-NEXT:    [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ]
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ]
; CHECK-NEXT:    ret ptr [[RES]]
;
; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit(
; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]]
; DISABLE:       [[HEADER_PREHEADER]]:
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_SUCC:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT:.*]], label %[[HEADER]]
; DISABLE:       [[EXIT_SUCC]]:
; DISABLE-NEXT:    [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    br label %[[EXIT:.*]]
; DISABLE:       [[EXIT_FAIL_LOOPEXIT]]:
; DISABLE-NEXT:    br label %[[EXIT_FAIL]]
; DISABLE:       [[EXIT_FAIL]]:
; DISABLE-NEXT:    [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ]
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT]]:
; DISABLE-NEXT:    [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ]
; DISABLE-NEXT:    ret ptr [[RES]]
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit_fail, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit_succ, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit_fail, label %header

exit_succ:
  %res_succ = phi ptr [ %search_ptr, %match_check ]
  br label %exit

exit_fail:
  %res_fail = phi ptr [ %search_end, %entry ], [ %search_end, %search_check ]
  br label %exit

exit:
  %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ]
  ret ptr %res
}

define ptr @ensure_not_found_successors_fixed(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @ensure_not_found_successors_fixed(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoaddr ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoaddr ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoaddr ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sub i64 [[TMP2]], [[TMP3]]
; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i8, ptr [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 16 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 16
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP6]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[FOUND_MATCH:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 16
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[FIND_FIRST_VEC_HEADER]], label %[[NOT_FOUND:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[FOUND_MATCH]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[NOT_FOUND]], label %[[HEADER]]
; CHECK:       [[FOUND_MATCH]]:
; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    ret ptr [[SEARCH_PTR_LCSSA]]
; CHECK:       [[NOT_FOUND]]:
; CHECK-NEXT:    [[UNUSED:%.*]] = phi i64 [ 0, %[[SEARCH_CHECK]] ], [ 0, %[[SEARCH_CHECK_VEC]] ]
; CHECK-NEXT:    ret ptr null
;
; DISABLE-LABEL: define ptr @ensure_not_found_successors_fixed(
; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[ENTRY]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[FOUND_MATCH:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[NOT_FOUND:.*]], label %[[HEADER]]
; DISABLE:       [[FOUND_MATCH]]:
; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    ret ptr [[SEARCH_PTR_LCSSA]]
; DISABLE:       [[NOT_FOUND]]:
; DISABLE-NEXT:    [[UNUSED:%.*]] = phi i64 [ 0, %[[SEARCH_CHECK]] ]
; DISABLE-NEXT:    ret ptr null
;
entry:
  br label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %found_match, label %needle_check

search_check:
  %search_next = getelementptr i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %not_found, label %header

found_match:
  ret ptr %search_ptr

not_found:
  %unused = phi i64 [ 0, %search_check ]
  ret ptr null
}

; This is a small variation of the previous test case, where we have another value
; in a successor phi:
;
;   entry:
;     %dummy_val = getelementptr i8, ptr %search_start, i64 1
;     br label %header
;   ...
;   not_found:
;     %unused = phi ptr [ %dummy_val, %search_check ]
;
define ptr @ensure_not_found_successors_fixed2(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @ensure_not_found_successors_fixed2(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[DUMMY_VAL:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 1
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoaddr ptr [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoaddr ptr [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoaddr ptr [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sub i64 [[TMP2]], [[TMP3]]
; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP4]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i8, ptr [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 16 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 16
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP6]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[FOUND_MATCH:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 16
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[FIND_FIRST_VEC_HEADER]], label %[[NOT_FOUND:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[FOUND_MATCH]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[NOT_FOUND]], label %[[HEADER]]
; CHECK:       [[FOUND_MATCH]]:
; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    ret ptr [[SEARCH_PTR_LCSSA]]
; CHECK:       [[NOT_FOUND]]:
; CHECK-NEXT:    [[UNUSED:%.*]] = phi ptr [ [[DUMMY_VAL]], %[[SEARCH_CHECK]] ], [ [[DUMMY_VAL]], %[[SEARCH_CHECK_VEC]] ]
; CHECK-NEXT:    ret ptr null
;
; DISABLE-LABEL: define ptr @ensure_not_found_successors_fixed2(
; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    [[DUMMY_VAL:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 1
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[ENTRY]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[FOUND_MATCH:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[NOT_FOUND:.*]], label %[[HEADER]]
; DISABLE:       [[FOUND_MATCH]]:
; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    ret ptr [[SEARCH_PTR_LCSSA]]
; DISABLE:       [[NOT_FOUND]]:
; DISABLE-NEXT:    [[UNUSED:%.*]] = phi ptr [ [[DUMMY_VAL]], %[[SEARCH_CHECK]] ]
; DISABLE-NEXT:    ret ptr null
;
entry:
  %dummy_val = getelementptr i8, ptr %search_start, i64 1
  br label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %found_match, label %needle_check

search_check:
  %search_next = getelementptr i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %not_found, label %header

found_match:
  ret ptr %search_ptr

not_found:
  %unused = phi ptr [ %dummy_val, %search_check ]
  ret ptr null
}

; Same as @find_first_of_i8 but mixing address spaces
; This verifies that the vectorized loop correctly preserves address spaces.
define ptr addrspace(1) @find_first_of_i8_addrspace(ptr addrspace(1) %search_start, ptr addrspace(1) %search_end, ptr addrspace(2) %needle_start, ptr addrspace(2) %needle_end) #0 {
; CHECK-LABEL: define ptr addrspace(1) @find_first_of_i8_addrspace(
; CHECK-SAME: ptr addrspace(1) [[SEARCH_START:%.*]], ptr addrspace(1) [[SEARCH_END:%.*]], ptr addrspace(2) [[NEEDLE_START:%.*]], ptr addrspace(2) [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr addrspace(1) [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr addrspace(2) [[NEEDLE_START]], [[NEEDLE_END]]
; CHECK-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; CHECK-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; CHECK:       [[HEADER_PREHEADER]]:
; CHECK-NEXT:    br label %[[MEM_CHECK:.*]]
; CHECK:       [[MEM_CHECK]]:
; CHECK-NEXT:    [[SEARCH_START_INT:%.*]] = ptrtoint ptr addrspace(1) [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_END_INT:%.*]] = ptrtoint ptr addrspace(1) [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoaddr ptr addrspace(1) [[SEARCH_END]] to i64
; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoaddr ptr addrspace(1) [[SEARCH_START]] to i64
; CHECK-NEXT:    [[SEARCH_TRIP_COUNT:%.*]] = sub i64 [[TMP6]], [[TMP1]]
; CHECK-NEXT:    [[NEEDLE_START_INT:%.*]] = ptrtoint ptr addrspace(2) [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_END_INT:%.*]] = ptrtoint ptr addrspace(2) [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoaddr ptr addrspace(2) [[NEEDLE_END]] to i64
; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoaddr ptr addrspace(2) [[NEEDLE_START]] to i64
; CHECK-NEXT:    [[NEEDLE_TRIP_COUNT:%.*]] = sub i64 [[TMP2]], [[TMP7]]
; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
; CHECK-NEXT:    [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
; CHECK-NEXT:    [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
; CHECK-NEXT:    [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
; CHECK-NEXT:    [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
; CHECK-NEXT:    [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
; CHECK-NEXT:    [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
; CHECK-NEXT:    [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
; CHECK-NEXT:    br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]]
; CHECK:       [[FIND_FIRST_VEC_HEADER]]:
; CHECK-NEXT:    [[SEARCH_IDX:%.*]] = phi i64 [ 0, %[[MEM_CHECK]] ], [ [[SEARCH_IDX_NEXT:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
; CHECK-NEXT:    [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[SEARCH_IDX]], i64 [[SEARCH_TRIP_COUNT]])
; CHECK-NEXT:    [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
; CHECK-NEXT:    [[PSEARCH:%.*]] = getelementptr i8, ptr addrspace(1) [[SEARCH_START]], i64 [[SEARCH_IDX]]
; CHECK-NEXT:    [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p1(ptr addrspace(1) align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    br label %[[NEEDLE_CHECK_VEC:.*]]
; CHECK:       [[NEEDLE_CHECK_VEC]]:
; CHECK-NEXT:    [[NEEDLE_IDX:%.*]] = phi i64 [ 0, %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_IDX_NEXT:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[PMATCH:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[FIND_FIRST_VEC_HEADER]] ], [ [[MATCH_ACCUMULATOR:%.*]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[NEEDLE_IDX]], i64 [[NEEDLE_TRIP_COUNT]])
; CHECK-NEXT:    [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
; CHECK-NEXT:    [[PNEEDLE:%.*]] = getelementptr i8, ptr addrspace(2) [[NEEDLE_START]], i64 [[NEEDLE_IDX]]
; CHECK-NEXT:    [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p2(ptr addrspace(2) align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
; CHECK-NEXT:    [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
; CHECK-NEXT:    [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT:    [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
; CHECK-NEXT:    [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
; CHECK-NEXT:    [[MATCH_SEGMENT:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
; CHECK-NEXT:    [[MATCH_ACCUMULATOR]] = or <vscale x 16 x i1> [[PMATCH]], [[MATCH_SEGMENT]]
; CHECK-NEXT:    [[NEEDLE_IDX_NEXT]] = add i64 [[NEEDLE_IDX]], 16
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[NEEDLE_IDX_NEXT]], [[NEEDLE_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP3]], label %[[NEEDLE_CHECK_VEC]], label %[[MATCH_CHECK_VEC:.*]]
; CHECK:       [[MATCH_CHECK_VEC]]:
; CHECK-NEXT:    [[MATCH_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_ACCUMULATOR]], %[[NEEDLE_CHECK_VEC]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
; CHECK-NEXT:    br i1 [[TMP4]], label %[[CALCULATE_MATCH:.*]], label %[[SEARCH_CHECK_VEC]]
; CHECK:       [[CALCULATE_MATCH]]:
; CHECK-NEXT:    [[MATCH_START:%.*]] = phi ptr addrspace(1) [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
; CHECK-NEXT:    [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
; CHECK-NEXT:    [[MATCH_RES:%.*]] = getelementptr i8, ptr addrspace(1) [[MATCH_START]], i64 [[MATCH_IDX]]
; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
; CHECK:       [[SEARCH_CHECK_VEC]]:
; CHECK-NEXT:    [[SEARCH_IDX_NEXT]] = add i64 [[SEARCH_IDX]], 16
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[SEARCH_IDX_NEXT]], [[SEARCH_TRIP_COUNT]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
; CHECK:       [[SCALAR_PREHEADER]]:
; CHECK-NEXT:    br label %[[HEADER:.*]]
; CHECK:       [[HEADER]]:
; CHECK-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr addrspace(1) [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
; CHECK-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr addrspace(1) [[SEARCH_PTR]], align 1
; CHECK-NEXT:    br label %[[MATCH_CHECK:.*]]
; CHECK:       [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr addrspace(2) [[NEEDLE_NEXT]], [[NEEDLE_END]]
; CHECK-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; CHECK:       [[MATCH_CHECK]]:
; CHECK-NEXT:    [[NEEDLE_PTR]] = phi ptr addrspace(2) [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr addrspace(2) [[NEEDLE_PTR]], align 1
; CHECK-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; CHECK-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
; CHECK:       [[SEARCH_CHECK]]:
; CHECK-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr addrspace(1) [[SEARCH_PTR]], i64 1
; CHECK-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr addrspace(1) [[SEARCH_NEXT]], [[SEARCH_END]]
; CHECK-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
; CHECK:       [[EXIT_LOOPEXIT]]:
; CHECK-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr addrspace(1) [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT_LOOPEXIT1]]:
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RES:%.*]] = phi ptr addrspace(1) [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT:    ret ptr addrspace(1) [[RES]]
;
; DISABLE-LABEL: define ptr addrspace(1) @find_first_of_i8_addrspace(
; DISABLE-SAME: ptr addrspace(1) [[SEARCH_START:%.*]], ptr addrspace(1) [[SEARCH_END:%.*]], ptr addrspace(2) [[NEEDLE_START:%.*]], ptr addrspace(2) [[NEEDLE_END:%.*]]) #[[ATTR0]] {
; DISABLE-NEXT:  [[ENTRY:.*]]:
; DISABLE-NEXT:    [[SEARCH_TEST:%.*]] = icmp eq ptr addrspace(1) [[SEARCH_START]], [[SEARCH_END]]
; DISABLE-NEXT:    [[NEEDLE_TEST:%.*]] = icmp eq ptr addrspace(2) [[NEEDLE_START]], [[NEEDLE_END]]
; DISABLE-NEXT:    [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
; DISABLE-NEXT:    br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
; DISABLE:       [[HEADER_PREHEADER]]:
; DISABLE-NEXT:    br label %[[HEADER:.*]]
; DISABLE:       [[HEADER]]:
; DISABLE-NEXT:    [[SEARCH_PTR:%.*]] = phi ptr addrspace(1) [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; DISABLE-NEXT:    [[SEARCH_LOAD:%.*]] = load i8, ptr addrspace(1) [[SEARCH_PTR]], align 1
; DISABLE-NEXT:    br label %[[MATCH_CHECK:.*]]
; DISABLE:       [[NEEDLE_CHECK:.*]]:
; DISABLE-NEXT:    [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[NEEDLE_PTR:%.*]], i64 1
; DISABLE-NEXT:    [[NEEDLE_CMP:%.*]] = icmp eq ptr addrspace(2) [[NEEDLE_NEXT]], [[NEEDLE_END]]
; DISABLE-NEXT:    br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
; DISABLE:       [[MATCH_CHECK]]:
; DISABLE-NEXT:    [[NEEDLE_PTR]] = phi ptr addrspace(2) [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; DISABLE-NEXT:    [[NEEDLE_LOAD:%.*]] = load i8, ptr addrspace(2) [[NEEDLE_PTR]], align 1
; DISABLE-NEXT:    [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
; DISABLE-NEXT:    br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
; DISABLE:       [[SEARCH_CHECK]]:
; DISABLE-NEXT:    [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr addrspace(1) [[SEARCH_PTR]], i64 1
; DISABLE-NEXT:    [[SEARCH_CMP:%.*]] = icmp eq ptr addrspace(1) [[SEARCH_NEXT]], [[SEARCH_END]]
; DISABLE-NEXT:    br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
; DISABLE:       [[EXIT_LOOPEXIT]]:
; DISABLE-NEXT:    [[SEARCH_PTR_LCSSA:%.*]] = phi ptr addrspace(1) [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT_LOOPEXIT1]]:
; DISABLE-NEXT:    br label %[[EXIT]]
; DISABLE:       [[EXIT]]:
; DISABLE-NEXT:    [[RES:%.*]] = phi ptr addrspace(1) [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; DISABLE-NEXT:    ret ptr addrspace(1) [[RES]]
;
entry:
  %search_test = icmp eq ptr addrspace(1) %search_start, %search_end
  %needle_test = icmp eq ptr addrspace(2) %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr addrspace(1) [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr addrspace(1) %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr addrspace(2) %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr addrspace(2) %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr addrspace(2) [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr addrspace(2) %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr addrspace(1) %search_ptr, i64 1
  %search_cmp = icmp eq ptr addrspace(1) %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr addrspace(1) [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  ret ptr addrspace(1) %res
}

; From here on we only test for the presence/absence of the intrinsic.
; UTC_ARGS: --disable

; Same as @find_first_of_i8 but with `ne' comparison.
; This is rejected for now, but should eventually be supported.
define ptr @find_first_not_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_not_of_i8(
; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
; DISABLE-LABEL: define ptr @find_first_not_of_i8(
; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp ne i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  ret ptr %res
}

; This is the same as @find_first_of_i8 but without SVE2, which we require to
; perform the conversion.
define ptr @find_first_of_i8_nosve2(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) {
; CHECK-LABEL: define ptr @find_first_of_i8_nosve2(
; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
; DISABLE-LABEL: define ptr @find_first_of_i8_nosve2(
; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  ret ptr %res
}

; Same as @find_first_of_i8 but here we use the inner PHI outside the loop nest.
; This isn't supported.
define ptr @find_first_of_i8_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_outside_use(
; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
; DISABLE-LABEL: define ptr @find_first_of_i8_outside_use(
; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit, label %header

exit:
  %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
  %use = phi ptr [ %needle_end, %entry ], [ %needle_ptr, %match_check ], [ %needle_end, %search_check ]
  ret ptr %res
}

; Same as @find_first_of_i8_multi_exit but `search_ptr' is used in `exit_fail'
; which should block the transform.
define ptr @find_first_of_i8_multi_exit_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use(
; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use(
; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}}
;
entry:
  %search_test = icmp eq ptr %search_start, %search_end
  %needle_test = icmp eq ptr %needle_start, %needle_end
  %combined_test = or i1 %search_test, %needle_test
  br i1 %combined_test, label %exit_fail, label %header

header:
  %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
  %search_load = load i8, ptr %search_ptr, align 1
  br label %match_check

needle_check:
  %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
  %needle_cmp = icmp eq ptr %needle_next, %needle_end
  br i1 %needle_cmp, label %search_check, label %match_check

match_check:
  %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
  %needle_load = load i8, ptr %needle_ptr, align 1
  %match_cmp = icmp eq i8 %search_load, %needle_load
  br i1 %match_cmp, label %exit_succ, label %needle_check

search_check:
  %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
  %search_cmp = icmp eq ptr %search_next, %search_end
  br i1 %search_cmp, label %exit_fail, label %header

exit_succ:
  %res_succ = phi ptr [ %search_ptr, %match_check ]
  br label %exit

exit_fail:
  %res_fail = phi ptr [ %search_end, %entry ], [ %search_ptr, %search_check ]
  br label %exit

exit:
  %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ]
  ret ptr %res
}

attributes #0 = { "target-features"="+sve2" }
;.
; CHECK: [[PROF0]] = !{!"branch_weights", i32 10, i32 90}
;.