derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
342 lines
14 KiB
LLVM
342 lines
14 KiB
LLVM
; Test branch weight metadata, estimated trip count metadata, and block
|
|
; frequencies after partial loop unrolling without -unroll-runtime such that
|
|
; some iterations' latches become unconditional, which often contradicts the
|
|
; original branch weights.
|
|
;
|
|
; (unroll-complete.ll tests complete loop unrolling, in which the final unrolled
|
|
; iteration unconditionally exits (backedge removed). Here, we include cases
|
|
; where the final iteration's latch unconditionally continues instead.)
|
|
;
|
|
; For each case, we check:
|
|
; - Iteration frequencies
|
|
; - When each is multiplied by the number of original loop bodies that execute
|
|
; within it, they should sum to almost exactly the original loop body
|
|
; frequency.
|
|
; - The only exception is an impossibly high or low original frequency (e.g.,
|
|
; due to bad profile data), for which there exist no new branch weights that
|
|
; can yield that frequency sum. In those cases, we expect the maximum or
|
|
; minimum possible frequency.
|
|
; - CFGs
|
|
; - We verify which branch weights go with which branches and that we did not
|
|
; overlook any other branch weights (no extra !prof or branch_weights).
|
|
; - We also check the number of original loop bodies (represented by a call to
|
|
; @f) that appear within each unrolled iteration.
|
|
; - Branch weight metadata
|
|
; - Checking frequencies already checks whether the branch weights have the
|
|
; expected effect, but we also want to check that we get uniform
|
|
; probabilities/weights (same !prof) across the unrolled iteration latches
|
|
; when expected.
|
|
; - llvm.loop.estimated_trip_count
|
|
; - It must be the number of iterations of the unrolled loop required for the
|
|
; original loop body to reach its original frequency.
|
|
; - It must not be blindly computed from any new latch branch weights.
|
|
|
|
; ------------------------------------------------------------------------------
|
|
; Define LIT substitutions.
|
|
;
|
|
; For verifying that the test code produces the original loop body frequency we
|
|
; expect.
|
|
; DEFINE: %{bf-fc} = opt %t.ll -S -passes='print<block-freq>' 2>&1 | \
|
|
; DEFINE: FileCheck %s -check-prefixes
|
|
;
|
|
; For checking the unrolled loop:
|
|
; DEFINE: %{ur-bf} = opt %t.ll -S -passes='loop-unroll,print<block-freq>' 2>&1
|
|
; DEFINE: %{fc} = FileCheck %s \
|
|
; DEFINE: -implicit-check-not='llvm.loop.estimated_trip_count' \
|
|
; DEFINE: -implicit-check-not='!prof' \
|
|
; DEFINE: -implicit-check-not='branch_weights' \
|
|
; DEFINE: -implicit-check-not='call void @f' -check-prefixes
|
|
|
|
; ------------------------------------------------------------------------------
|
|
; Check cases when the original loop's number of iterations is a run-time
|
|
; determined multiple of 10 and the original loop body frequency is 10.
|
|
;
|
|
; RUN: sed -e s/@N@/%mul10/ -e s/@W@/9/ %s > %t.ll
|
|
;
|
|
; At compile time, possibilities for that value always include unroll count x 10
|
|
; x N for any integer N >= 1, so the unrolled loop's backedge always remains
|
|
; conditional, so we check cases where it becomes unconditional later in this
|
|
; test file with the CONST4 config.
|
|
;
|
|
; Check the original loop body frequency.
|
|
;
|
|
; RUN: %{bf-fc} MULT-ORIG
|
|
; MULT-ORIG: - do.body: float = 10.0,
|
|
;
|
|
; When the unroll count is odd, every iteration's latch remains conditional, so
|
|
; their original probabilities are not contradicted. That is, the original loop
|
|
; latch's branch weights remain on all unrolled iterations' latches.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=3 | %{fc} MULT3
|
|
;
|
|
; Sums to approximately the original loop body frequency, 10.
|
|
; MULT3: - do.body: float = 3.69,
|
|
; MULT3: - do.body.1: float = 3.321,
|
|
; MULT3: - do.body.2: float = 2.9889,
|
|
;
|
|
; MULT3: call void @f
|
|
; MULT3: br i1 %{{.*}}, label %do.body.1, label %do.end, !prof !0
|
|
; MULT3: call void @f
|
|
; MULT3: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; MULT3: call void @f
|
|
; MULT3: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1
|
|
;
|
|
; MULT3: !0 = !{!"branch_weights", i32 9, i32 1}
|
|
; MULT3: !1 = distinct !{!1, !2, !3}
|
|
; MULT3: !2 = !{!"llvm.loop.estimated_trip_count", i32 4}
|
|
; MULT3: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; When the unroll count is even, odd-numbered unrolled iterations become
|
|
; unconditional, so branch weights must be adjusted.
|
|
;
|
|
; -unroll-count=2, so there is 1 remaining conditional latch, so the
|
|
; implementation can compute uniform weights by solving a linear equation.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=2 | %{fc} MULT2
|
|
;
|
|
; Multiply by 2 to get the original loop body frequency, 10.
|
|
; MULT2: - do.body: float = 5.0,
|
|
;
|
|
; MULT2: call void @f
|
|
; MULT2-NOT: br
|
|
; MULT2: call void @f
|
|
; MULT2: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1{{$}}
|
|
;
|
|
; The branch weights imply the estimated trip count is
|
|
; (1717986918+429496730)/429496730 = approximately (8+2)/2 = 5.
|
|
; MULT2: !0 = !{!"branch_weights", i32 1717986918, i32 429496730}
|
|
; MULT2: !1 = distinct !{!1, !2, !3}
|
|
; MULT2: !2 = !{!"llvm.loop.estimated_trip_count", i32 5}
|
|
; MULT2: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; -unroll-count=4, so there are 2 remaining conditional latches, so the
|
|
; implementation can compute uniform weights using the quadratic formula.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=4 | %{fc} MULT4
|
|
;
|
|
; Multiply by 2 and sum to get the original loop body frequency, 10.
|
|
; MULT4: - do.body: float = 2.7778,
|
|
; MULT4: - do.body.2: float = 2.2222,
|
|
;
|
|
; MULT4: call void @f
|
|
; MULT4-NOT: br
|
|
; MULT4: call void @f
|
|
; MULT4: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; MULT4: call void @f
|
|
; MULT4-NOT: br
|
|
; MULT4: call void @f
|
|
; MULT4: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1
|
|
;
|
|
; MULT4 is like applying -unroll-count=2 to MULT2 without converting any
|
|
; more conditional latches to unconditional, so MULT2's branch weights work.
|
|
; MULT4: !0 = !{!"branch_weights", i32 1717986918, i32 429496730}
|
|
; MULT4: !1 = distinct !{!1, !2, !3}
|
|
; MULT4: !2 = !{!"llvm.loop.estimated_trip_count", i32 3}
|
|
; MULT4: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; -unroll-count=6, so there are 3 remaining conditional latches, the lowest
|
|
; number where the implementation cannot compute uniform weights using a
|
|
; simple formula.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=6 | %{fc} MULT6
|
|
;
|
|
; Multiply by 2 and sum to get the original loop body frequency, 10.
|
|
; MULT6: - do.body: float = 2.1956,
|
|
; MULT6: - do.body.2: float = 1.476,
|
|
; MULT6: - do.body.4: float = 1.3284,
|
|
;
|
|
; MULT6: call void @f
|
|
; MULT6-NOT: br
|
|
; MULT6: call void @f
|
|
; MULT6: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; MULT6: call void @f
|
|
; MULT6-NOT: br
|
|
; MULT6: call void @f
|
|
; MULT6: br i1 %{{.*}}, label %do.body.4, label %do.end, !prof !1
|
|
; MULT6: call void @f
|
|
; MULT6-NOT: br
|
|
; MULT6: call void @f
|
|
; MULT6: br i1 %{{.*}}, label %do.body, label %do.end, !prof !1, !llvm.loop !2
|
|
;
|
|
; There are 3 conditional latches remaining, so it adjusts the first and
|
|
; leaves the second two with the original loop's branch weights.
|
|
; MULT6: !0 = !{!"branch_weights", i32 1443686486, i32 703797162}
|
|
; MULT6: !1 = !{!"branch_weights", i32 9, i32 1}
|
|
; MULT6: !2 = distinct !{!2, !3, !4}
|
|
; MULT6: !3 = !{!"llvm.loop.estimated_trip_count", i32 2}
|
|
; MULT6: !4 = !{!"llvm.loop.unroll.disable"}
|
|
|
|
; ------------------------------------------------------------------------------
|
|
; Check case when the original loop's number of iterations is a run-time
|
|
; determined multiple of 10, the unroll count is even so that odd-numbered
|
|
; unrolled iterations become unconditional, and the original loop body frequency
|
|
; is 1, which is impossibly low. This case is important to ensure the
|
|
; implementation does not malfunction by trying to use negative and possibly
|
|
; infinite probabilities to reach the original loop body frequency.
|
|
;
|
|
; RUN: sed -e s/@N@/%mul10/ -e s/@W@/0/ %s > %t.ll
|
|
;
|
|
; Check the original loop body frequency.
|
|
;
|
|
; RUN: %{bf-fc} LOW-ORIG
|
|
; LOW-ORIG: - do.body: float = 1.0,
|
|
;
|
|
; -unroll-count=2, so there is 1 remaining conditional latch. The
|
|
; implementation tries to compute uniform weights by solving a linear equation
|
|
; but ultimately sets the latch's probability to zero.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=2 | %{fc} LOW2
|
|
;
|
|
; Multiply by 2, but the result is greater than the original loop body
|
|
; frequency, 1, which is impossibly low.
|
|
; LOW2: - do.body: float = 1.0,
|
|
;
|
|
; LOW2: call void @f
|
|
; LOW2-NOT: br
|
|
; LOW2: call void @f
|
|
; LOW2: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1{{$}}
|
|
;
|
|
; LOW2: !0 = !{!"branch_weights", i32 0, i32 -2147483648}
|
|
; LOW2: !1 = distinct !{!1, !2, !3}
|
|
; LOW2: !2 = !{!"llvm.loop.estimated_trip_count", i32 1}
|
|
; LOW2: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; -unroll-count=4, so there are 2 remaining conditional latches. The
|
|
; implementation tries to compute uniform weights using the quadratic formula
|
|
; but ultimately sets both latches' probabilities to zero.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=4 | %{fc} LOW4
|
|
;
|
|
; Multiply by 2 and sum, but the result is greater than the original loop body
|
|
; frequency, 1, which is impossibly low.
|
|
; LOW4: - do.body: float = 1.0,
|
|
; LOW4: - do.body.2: float = 0.0{{(0000[0-9]*)?}},
|
|
;
|
|
; LOW4: call void @f
|
|
; LOW4-NOT: br
|
|
; LOW4: call void @f
|
|
; LOW4: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; LOW4: call void @f
|
|
; LOW4-NOT: br
|
|
; LOW4: call void @f
|
|
; LOW4: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1
|
|
;
|
|
; LOW4: !0 = !{!"branch_weights", i32 0, i32 -2147483648}
|
|
; LOW4: !1 = distinct !{!1, !2, !3}
|
|
; LOW4: !2 = !{!"llvm.loop.estimated_trip_count", i32 1}
|
|
; LOW4: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; -unroll-count=6, so there are 3 remaining conditional latches. The
|
|
; implementation cannot compute uniform weights using a simple formula, and
|
|
; ultimately it must set all those latches' probabilities to zero. The
|
|
; implementation will face a new stumbling block starting at the second latch:
|
|
; reaching the remaining iterations already has a zero probability due to the
|
|
; zero probability set at the first latch, so the required probability could
|
|
; accidentally be computed as negative infinity.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=6 | %{fc} LOW6
|
|
;
|
|
; Multiply by 2 and sum, but the result is greater than the original loop body
|
|
; frequency, 1, which is impossibly low.
|
|
; LOW6: - do.body: float = 1.0,
|
|
; LOW6: - do.body.2: float = 0.0{{(0000[0-9]*)?}},
|
|
; LOW6: - do.body.4: float = 0.0{{(0000[0-9]*)?}},
|
|
;
|
|
; LOW6: call void @f
|
|
; LOW6-NOT: br
|
|
; LOW6: call void @f
|
|
; LOW6: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; LOW6: call void @f
|
|
; LOW6-NOT: br
|
|
; LOW6: call void @f
|
|
; LOW6: br i1 %{{.*}}, label %do.body.4, label %do.end, !prof !0
|
|
; LOW6: call void @f
|
|
; LOW6-NOT: br
|
|
; LOW6: call void @f
|
|
; LOW6: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1
|
|
;
|
|
; LOW6: !0 = !{!"branch_weights", i32 0, i32 -2147483648}
|
|
; LOW6: !1 = distinct !{!1, !2, !3}
|
|
; LOW6: !2 = !{!"llvm.loop.estimated_trip_count", i32 1}
|
|
; LOW6: !3 = !{!"llvm.loop.unroll.disable"}
|
|
|
|
; ------------------------------------------------------------------------------
|
|
; Check cases when the original loop's number of iterations is a constant 10 and
|
|
; the original loop body frequency is 10.
|
|
;
|
|
; RUN: sed -e s/@N@/10/g -e s/@W@/9/ %s > %t.ll
|
|
;
|
|
; Because we test only partial unrolling, there is always exactly one unrolled
|
|
; iteration that can possibly exit, so only its latch can remain conditional.
|
|
; Because there is only one, its branch weights can be computed with a simple
|
|
; formula.
|
|
;
|
|
; Check the original loop body frequency.
|
|
;
|
|
; RUN: %{bf-fc} CONST-ORIG
|
|
; CONST-ORIG: - do.body: float = 10.0,
|
|
;
|
|
; Check when the unrolled loop's backedge remains conditional.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=2 | %{fc} CONST2
|
|
;
|
|
; Multiply by 2 to get the original loop body frequency, 10.
|
|
; CONST2: - do.body: float = 5.0,
|
|
;
|
|
; CONST2: call void @f
|
|
; CONST2-NOT: br:
|
|
; CONST2: call void @f
|
|
; CONST2: br i1 %{{.*}}, label %do.body, label %do.end, !prof !0, !llvm.loop !1
|
|
;
|
|
; Like MULT2.
|
|
; CONST2: !0 = !{!"branch_weights", i32 1717986918, i32 429496730}
|
|
; CONST2: !1 = distinct !{!1, !2, !3}
|
|
; CONST2: !2 = !{!"llvm.loop.estimated_trip_count", i32 5}
|
|
; CONST2: !3 = !{!"llvm.loop.unroll.disable"}
|
|
;
|
|
; Check when the unrolled loop's backedge unconditionally continues.
|
|
;
|
|
; RUN: %{ur-bf} -unroll-count=4 | %{fc} CONST4
|
|
;
|
|
; Multiply by 2 and sum to get the original loop body frequency, 10.
|
|
; CONST4: - do.body: float = 3.0,
|
|
; CONST4: - do.body.2: float = 2.0,
|
|
;
|
|
; CONST4: call void @f
|
|
; CONST4-NOT: br
|
|
; CONST4: call void @f
|
|
; CONST4: br i1 %{{.*}}, label %do.body.2, label %do.end, !prof !0
|
|
; CONST4: call void @f
|
|
; CONST4-NOT: br
|
|
; CONST4: call void @f
|
|
; CONST4: br label %do.body, !llvm.loop !1
|
|
;
|
|
; There is no llvm.loop.estimated_trip_count because the unrolled loop's latch
|
|
; in do.body.2 unconditionally continues. The branch weights on do.body's
|
|
; branch imply do.body continues twice and then exits once, thus executing the
|
|
; original loop body 10 times.
|
|
; CONST4: !0 = !{!"branch_weights", i32 1431655765, i32 715827883}
|
|
; CONST4: !1 = distinct !{!1, !2}
|
|
; CONST4: !2 = !{!"llvm.loop.unroll.disable"}
|
|
|
|
declare void @f(i32)
|
|
|
|
define void @test(i32 %n) {
|
|
entry:
|
|
%mul10 = mul i32 %n, 10
|
|
br label %do.body
|
|
|
|
do.body:
|
|
%i = phi i32 [ 0, %entry ], [ %next, %do.body ]
|
|
call void @f(i32 %i)
|
|
%next = add i32 %i, 1
|
|
%c = icmp ne i32 %next, @N@
|
|
br i1 %c, label %do.body, label %do.end, !prof !0
|
|
|
|
do.end:
|
|
ret void
|
|
}
|
|
|
|
; Loop body frequency is @W@ + 1.
|
|
!0 = !{!"branch_weights", i32 @W@, i32 1}
|