derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
325 lines
18 KiB
LLVM
325 lines
18 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: opt < %s -passes=loop-vectorize -S -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a320 2>&1 | FileCheck %s --check-prefix=A320
|
|
|
|
; The loop below is a small counted loop with a scalar reduction.
|
|
; When AggressiveInterleaving is enabled for this subtarget, LoopVectorize
|
|
; should choose an interleave count > 1 (VF == 1), which manifests as
|
|
; multiple loads / multiplies / adds in the vector body.
|
|
|
|
|
|
define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
|
|
; A320-LABEL: define void @test_interleave_reduction(
|
|
; A320-SAME: ptr [[ARG:%.*]], ptr [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
|
|
; A320-NEXT: [[ENTRY:.*:]]
|
|
; A320-NEXT: [[TPM15:%.*]] = load ptr, ptr [[ARG]], align 8
|
|
; A320-NEXT: [[TPM19:%.*]] = load ptr, ptr [[ARG1]], align 8
|
|
; A320-NEXT: br label %[[OUTER:.*]]
|
|
; A320: [[OUTER]]:
|
|
; A320-NEXT: [[TPM26:%.*]] = add i64 0, 1
|
|
; A320-NEXT: [[TPM10:%.*]] = alloca i32, align 8
|
|
; A320-NEXT: [[TPM102:%.*]] = ptrtoint ptr [[TPM10]] to i64
|
|
; A320-NEXT: [[TPM27:%.*]] = getelementptr inbounds i32, ptr [[TPM10]], i64 [[TPM26]]
|
|
; A320-NEXT: [[TPM28:%.*]] = getelementptr inbounds ptr, ptr [[TPM15]], i64 0
|
|
; A320-NEXT: [[TPM29:%.*]] = load ptr, ptr [[TPM28]], align 8
|
|
; A320-NEXT: [[TPM291:%.*]] = ptrtoint ptr [[TPM29]] to i64
|
|
; A320-NEXT: [[TPM17:%.*]] = alloca double, align 8
|
|
; A320-NEXT: [[TPM32:%.*]] = getelementptr inbounds double, ptr [[TPM17]], i64 [[TPM26]]
|
|
; A320-NEXT: [[TMP0:%.*]] = add i64 [[TPM291]], -8
|
|
; A320-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[TPM102]]
|
|
; A320-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
|
|
; A320-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
|
|
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
|
|
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; A320: [[VECTOR_PH]]:
|
|
; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
|
|
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
|
|
; A320-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
|
|
; A320-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP4]]
|
|
; A320-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
|
|
; A320-NEXT: [[IND_END3:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP5]]
|
|
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; A320: [[VECTOR_BODY]]:
|
|
; A320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI5:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
|
|
; A320-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 4
|
|
; A320-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[OFFSET_IDX]]
|
|
; A320-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP7]]
|
|
; A320-NEXT: [[OFFSET_IDX7:%.*]] = shl i64 [[INDEX]], 3
|
|
; A320-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX7]], 8
|
|
; A320-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[OFFSET_IDX7]]
|
|
; A320-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP9]]
|
|
; A320-NEXT: [[TMP10:%.*]] = load double, ptr [[NEXT_GEP8]], align 8
|
|
; A320-NEXT: [[TMP11:%.*]] = load double, ptr [[NEXT_GEP9]], align 8
|
|
; A320-NEXT: [[TMP12:%.*]] = load i32, ptr [[NEXT_GEP]], align 4
|
|
; A320-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP6]], align 4
|
|
; A320-NEXT: [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
|
|
; A320-NEXT: [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
|
|
; A320-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP14]]
|
|
; A320-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP15]]
|
|
; A320-NEXT: [[TMP18:%.*]] = load double, ptr [[TMP16]], align 8
|
|
; A320-NEXT: [[TMP19:%.*]] = load double, ptr [[TMP17]], align 8
|
|
; A320-NEXT: [[TMP20:%.*]] = fmul fast double [[TMP18]], [[TMP10]]
|
|
; A320-NEXT: [[TMP21:%.*]] = fmul fast double [[TMP19]], [[TMP11]]
|
|
; A320-NEXT: [[TMP22]] = fadd fast double [[TMP20]], [[VEC_PHI]]
|
|
; A320-NEXT: [[TMP23]] = fadd fast double [[TMP21]], [[VEC_PHI5]]
|
|
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
|
|
; A320-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; A320: [[MIDDLE_BLOCK]]:
|
|
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast double [[TMP23]], [[TMP22]]
|
|
; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER:.*]], label %[[SCALAR_PH]]
|
|
; A320: [[SCALAR_PH]]:
|
|
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TPM27]], %[[OUTER]] ]
|
|
; A320-NEXT: [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END3]], %[[MIDDLE_BLOCK]] ], [ [[TPM32]], %[[OUTER]] ]
|
|
; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[OUTER]] ]
|
|
; A320-NEXT: br label %[[INNER:.*]]
|
|
; A320: [[INNER]]:
|
|
; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ]
|
|
; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; A320-NEXT: [[TPM44:%.*]] = load double, ptr [[PHI_PTR_F64]], align 8
|
|
; A320-NEXT: [[TPM45:%.*]] = load i32, ptr [[PHI_PTR_I32]], align 4
|
|
; A320-NEXT: [[TPM46:%.*]] = zext i32 [[TPM45]] to i64
|
|
; A320-NEXT: [[TPM47:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TPM46]]
|
|
; A320-NEXT: [[TPM48:%.*]] = load double, ptr [[TPM47]], align 8
|
|
; A320-NEXT: [[TPM49:%.*]] = fmul fast double [[TPM48]], [[TPM44]]
|
|
; A320-NEXT: [[TPM50]] = fadd fast double [[TPM49]], [[PHI_ACC]]
|
|
; A320-NEXT: [[NEXT_I32]] = getelementptr inbounds i32, ptr [[PHI_PTR_I32]], i64 1
|
|
; A320-NEXT: [[NEXT_F64]] = getelementptr inbounds double, ptr [[PHI_PTR_F64]], i64 1
|
|
; A320-NEXT: [[DONE:%.*]] = icmp eq ptr [[NEXT_I32]], [[TPM29]]
|
|
; A320-NEXT: br i1 [[DONE]], label %[[EXIT_INNER]], label %[[INNER]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; A320: [[EXIT_INNER]]:
|
|
; A320-NEXT: [[TPM50_LCSSA:%.*]] = phi double [ [[TPM50]], %[[INNER]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; A320-NEXT: [[TPM35:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 0
|
|
; A320-NEXT: [[TPM37:%.*]] = fsub fast double 0.000000e+00, [[TPM50_LCSSA]]
|
|
; A320-NEXT: store double [[TPM37]], ptr [[TPM35]], align 8
|
|
; A320-NEXT: br label %[[OUTER]]
|
|
;
|
|
entry:
|
|
%tpm15 = load ptr, ptr %arg, align 8
|
|
%tpm19 = load ptr, ptr %arg1, align 8
|
|
br label %outer
|
|
|
|
outer: ; preds = %inner, %entry
|
|
%tpm26 = add i64 0, 1
|
|
%tpm10 = alloca i32, align 8
|
|
%tpm27 = getelementptr inbounds i32, ptr %tpm10, i64 %tpm26
|
|
%tpm28 = getelementptr inbounds ptr, ptr %tpm15, i64 0
|
|
%tpm29 = load ptr, ptr %tpm28, align 8
|
|
%tpm17 = alloca double, align 8
|
|
%tpm32 = getelementptr inbounds double, ptr %tpm17, i64 %tpm26
|
|
br label %inner
|
|
|
|
inner: ; preds = %inner, %outer
|
|
%phi.ptr.i32 = phi ptr [ %next.i32, %inner ], [ %tpm27, %outer ]
|
|
%phi.ptr.f64 = phi ptr [ %next.f64, %inner ], [ %tpm32, %outer ]
|
|
%phi.acc = phi double [ %tpm50, %inner ], [ 0.0, %outer ]
|
|
|
|
%tpm44 = load double, ptr %phi.ptr.f64, align 8
|
|
%tpm45 = load i32, ptr %phi.ptr.i32, align 4
|
|
%tpm46 = zext i32 %tpm45 to i64
|
|
%tpm47 = getelementptr inbounds double, ptr %tpm19, i64 %tpm46
|
|
%tpm48 = load double, ptr %tpm47, align 8
|
|
%tpm49 = fmul fast double %tpm48, %tpm44
|
|
%tpm50 = fadd fast double %tpm49, %phi.acc
|
|
|
|
%next.i32 = getelementptr inbounds i32, ptr %phi.ptr.i32, i64 1
|
|
%next.f64 = getelementptr inbounds double, ptr %phi.ptr.f64, i64 1
|
|
%done = icmp eq ptr %next.i32, %tpm29
|
|
br i1 %done, label %exit.inner, label %inner
|
|
|
|
exit.inner: ; preds = %inner
|
|
%tpm35 = getelementptr inbounds double, ptr %tpm19, i64 0
|
|
%tpm37 = fsub fast double 0.0, %tpm50
|
|
store double %tpm37, ptr %tpm35, align 8
|
|
br label %outer
|
|
}
|
|
|
|
;===---------------------------------------------------------------------===;
|
|
; 1) Simple sum-reduction over one array
|
|
; Expect: VF = 1 with interleave count > 1, so vector.body contains
|
|
; duplicated loads and adds.
|
|
;===---------------------------------------------------------------------===;
|
|
|
|
define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
|
|
; A320-LABEL: define double @sum_reduction(
|
|
; A320-SAME: ptr readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
|
|
; A320-NEXT: [[ENTRY:.*]]:
|
|
; A320-NEXT: [[CMP0:%.*]] = icmp eq i64 [[N]], 0
|
|
; A320-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
|
|
; A320: [[LOOP_PREHEADER]]:
|
|
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; A320: [[VECTOR_PH]]:
|
|
; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; A320: [[VECTOR_BODY]]:
|
|
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI1:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP1]]
|
|
; A320-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 2
|
|
; A320-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
|
|
; A320-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP6]], align 8
|
|
; A320-NEXT: [[TMP2]] = fadd fast <2 x double> [[VEC_PHI]], [[WIDE_LOAD]]
|
|
; A320-NEXT: [[TMP4]] = fadd fast <2 x double> [[VEC_PHI1]], [[WIDE_LOAD2]]
|
|
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 4
|
|
; A320-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
|
|
; A320: [[MIDDLE_BLOCK]]:
|
|
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP2]]
|
|
; A320-NEXT: [[TMP5:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
|
|
; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
|
|
; A320: [[SCALAR_PH]]:
|
|
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
|
|
; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
|
|
; A320-NEXT: br label %[[LOOP:.*]]
|
|
; A320: [[LOOP]]:
|
|
; A320-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; A320-NEXT: [[SUM:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
|
|
; A320-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
|
|
; A320-NEXT: [[VAL:%.*]] = load double, ptr [[GEP]], align 8
|
|
; A320-NEXT: [[SUM_NEXT]] = fadd fast double [[SUM]], [[VAL]]
|
|
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
|
|
; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP5:![0-9]+]]
|
|
; A320: [[EXIT_LOOPEXIT]]:
|
|
; A320-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi double [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
|
|
; A320-NEXT: br label %[[EXIT]]
|
|
; A320: [[EXIT]]:
|
|
; A320-NEXT: [[RES:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
|
|
; A320-NEXT: ret double [[RES]]
|
|
;
|
|
entry:
|
|
%cmp0 = icmp eq i64 %n, 0
|
|
br i1 %cmp0, label %exit, label %loop.preheader
|
|
|
|
loop.preheader:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 0, %loop.preheader ], [ %iv.next, %loop ]
|
|
%sum = phi double [ 0.0, %loop.preheader ], [ %sum.next, %loop ]
|
|
|
|
%gep = getelementptr inbounds double, ptr %a, i64 %iv
|
|
%val = load double, ptr %gep, align 8
|
|
%sum.next = fadd fast double %sum, %val
|
|
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%cond = icmp ult i64 %iv.next, %n
|
|
br i1 %cond, label %loop, label %exit
|
|
|
|
exit:
|
|
%res = phi double [ 0.0, %entry ], [ %sum.next, %loop ]
|
|
ret double %res
|
|
}
|
|
|
|
;===---------------------------------------------------------------------===;
|
|
; 2) Dot-product of two arrays
|
|
; Expect: again, VF = 1 with interleave count > 1. The vector body should
|
|
; have multiple pairs of loads and fmuls/fadds.
|
|
;===---------------------------------------------------------------------===;
|
|
|
|
define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b, i64 %n) {
|
|
; A320-LABEL: define double @dot_product(
|
|
; A320-SAME: ptr readonly captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
|
|
; A320-NEXT: [[ENTRY:.*]]:
|
|
; A320-NEXT: [[CMP0:%.*]] = icmp eq i64 [[N]], 0
|
|
; A320-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
|
|
; A320: [[LOOP_PREHEADER]]:
|
|
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; A320: [[VECTOR_PH]]:
|
|
; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; A320: [[VECTOR_BODY]]:
|
|
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[VEC_PHI1:%.*]] = phi <2 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; A320-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP1]]
|
|
; A320-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP1]]
|
|
; A320-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 2
|
|
; A320-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
|
|
; A320-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
|
|
; A320-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 2
|
|
; A320-NEXT: [[WIDE_LOAD3:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
|
|
; A320-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x double>, ptr [[TMP8]], align 8
|
|
; A320-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[WIDE_LOAD]], [[WIDE_LOAD3]]
|
|
; A320-NEXT: [[TMP10:%.*]] = fmul fast <2 x double> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
|
|
; A320-NEXT: [[TMP6]] = fadd fast <2 x double> [[VEC_PHI]], [[TMP4]]
|
|
; A320-NEXT: [[TMP7]] = fadd fast <2 x double> [[VEC_PHI1]], [[TMP10]]
|
|
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 4
|
|
; A320-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
; A320: [[MIDDLE_BLOCK]]:
|
|
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP6]]
|
|
; A320-NEXT: [[TMP9:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
|
|
; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
|
|
; A320: [[SCALAR_PH]]:
|
|
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
|
|
; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[LOOP_PREHEADER]] ]
|
|
; A320-NEXT: br label %[[LOOP:.*]]
|
|
; A320: [[LOOP]]:
|
|
; A320-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; A320-NEXT: [[ACC:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
|
|
; A320-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
|
|
; A320-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]]
|
|
; A320-NEXT: [[VA:%.*]] = load double, ptr [[GEP_A]], align 8
|
|
; A320-NEXT: [[VB:%.*]] = load double, ptr [[GEP_B]], align 8
|
|
; A320-NEXT: [[PROD:%.*]] = fmul fast double [[VA]], [[VB]]
|
|
; A320-NEXT: [[ACC_NEXT]] = fadd fast double [[ACC]], [[PROD]]
|
|
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
|
|
; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; A320: [[EXIT_LOOPEXIT]]:
|
|
; A320-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi double [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
|
|
; A320-NEXT: br label %[[EXIT]]
|
|
; A320: [[EXIT]]:
|
|
; A320-NEXT: [[RES:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[ACC_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
|
|
; A320-NEXT: ret double [[RES]]
|
|
;
|
|
entry:
|
|
%cmp0 = icmp eq i64 %n, 0
|
|
br i1 %cmp0, label %exit, label %loop.preheader
|
|
|
|
loop.preheader:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 0, %loop.preheader ], [ %iv.next, %loop ]
|
|
%acc = phi double [ 0.0, %loop.preheader ], [ %acc.next, %loop ]
|
|
|
|
%gep.a = getelementptr inbounds double, ptr %a, i64 %iv
|
|
%gep.b = getelementptr inbounds double, ptr %b, i64 %iv
|
|
%va = load double, ptr %gep.a, align 8
|
|
%vb = load double, ptr %gep.b, align 8
|
|
|
|
%prod = fmul fast double %va, %vb
|
|
%acc.next = fadd fast double %acc, %prod
|
|
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%cond = icmp ult i64 %iv.next, %n
|
|
br i1 %cond, label %loop, label %exit
|
|
|
|
exit:
|
|
%res = phi double [ 0.0, %entry ], [ %acc.next, %loop ]
|
|
ret double %res
|
|
}
|
|
;.
|
|
; A320: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
|
|
; A320: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
|
|
; A320: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
|
|
; A320: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
|
|
; A320: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
|
|
; A320: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
|
|
; A320: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
|
|
; A320: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
|
|
;.
|