derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
561 lines
22 KiB
LLVM
561 lines
22 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
|
|
; RUN: llc %s -o - | FileCheck %s --check-prefixes=COMMON,BASE
|
|
;; Additional runlines to exercise lsr code which AArch64 normally wouldn't.
|
|
; RUN: llc %s -o - -lsr-preferred-addressing-mode=preindexed | FileCheck %s --check-prefixes=COMMON,PREINDEX
|
|
; RUN: llc %s -o - -lsr-preferred-addressing-mode=postindexed | FileCheck %s --check-prefixes=COMMON,POSTINDEX
|
|
|
|
target triple = "aarch64-unknown-linux-gnu"
|
|
|
|
define void @mulvl123_addressing(ptr %src, ptr %dst, i64 %count) #0 {
|
|
; COMMON-LABEL: mulvl123_addressing:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: ptrue p0.b
|
|
; COMMON-NEXT: mov x8, xzr
|
|
; COMMON-NEXT: .LBB0_1: // %for.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: ldr z0, [x0]
|
|
; COMMON-NEXT: ldr z1, [x0, #1, mul vl]
|
|
; COMMON-NEXT: ldr z2, [x0, #2, mul vl]
|
|
; COMMON-NEXT: ldr z3, [x0, #3, mul vl]
|
|
; COMMON-NEXT: addvl x0, x0, #5
|
|
; COMMON-NEXT: umax z0.b, p0/m, z0.b, z1.b
|
|
; COMMON-NEXT: umax z2.b, p0/m, z2.b, z3.b
|
|
; COMMON-NEXT: umax z0.b, p0/m, z0.b, z2.b
|
|
; COMMON-NEXT: st1b { z0.b }, p0, [x1, x8]
|
|
; COMMON-NEXT: incb x8
|
|
; COMMON-NEXT: cmp x8, x2
|
|
; COMMON-NEXT: b.lo .LBB0_1
|
|
; COMMON-NEXT: // %bb.2: // %for.exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%2 = shl nuw nsw i64 %vscale, 4
|
|
%mul = shl nuw nsw i64 %vscale, 6
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%src.addr = phi ptr [ %src, %entry ], [ %src.addr.next, %for.body ]
|
|
%idx = phi i64 [ 0, %entry ], [ %idx.next, %for.body ]
|
|
%arrayidx = getelementptr inbounds i8, ptr %src.addr, i64 %idx
|
|
%3 = load <vscale x 16 x i8>, ptr %arrayidx
|
|
%4 = getelementptr <vscale x 16 x i8>, ptr %arrayidx, i64 1
|
|
%5 = load <vscale x 16 x i8>, ptr %4
|
|
%6 = getelementptr <vscale x 16 x i8>, ptr %arrayidx, i64 2
|
|
%7 = load <vscale x 16 x i8>, ptr %6
|
|
%8 = getelementptr <vscale x 16 x i8>, ptr %arrayidx, i64 3
|
|
%9 = load <vscale x 16 x i8>, ptr %8
|
|
%10 = tail call <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8> %3, <vscale x 16 x i8> %5)
|
|
%11 = tail call <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8> %7, <vscale x 16 x i8> %9)
|
|
%12 = tail call <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8> %10, <vscale x 16 x i8> %11)
|
|
%src.addr.next = getelementptr inbounds i8, ptr %src.addr, i64 %mul
|
|
%arrayidx4 = getelementptr inbounds i8, ptr %dst, i64 %idx
|
|
store <vscale x 16 x i8> %12, ptr %arrayidx4
|
|
%idx.next = add i64 %idx, %2
|
|
%cmp = icmp ult i64 %idx.next, %count
|
|
br i1 %cmp, label %for.body, label %for.exit
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
define void @many_mulvl1_addressing(ptr %src_rows, ptr %dst_rows, i64 %stride, i64 %count) #0 {
|
|
; COMMON-LABEL: many_mulvl1_addressing:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: ptrue p0.b
|
|
; COMMON-NEXT: ptrue p1.h
|
|
; COMMON-NEXT: .LBB1_1: // %for.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: add x8, x0, x2
|
|
; COMMON-NEXT: ldr z0, [x0]
|
|
; COMMON-NEXT: ld1b { z1.b }, p0/z, [x0, x2]
|
|
; COMMON-NEXT: ldr z2, [x0, #1, mul vl]
|
|
; COMMON-NEXT: ldr z3, [x8, #1, mul vl]
|
|
; COMMON-NEXT: incb x0, all, mul #2
|
|
; COMMON-NEXT: subs x3, x3, #1
|
|
; COMMON-NEXT: add z0.b, z0.b, z1.b
|
|
; COMMON-NEXT: add z1.b, z2.b, z3.b
|
|
; COMMON-NEXT: st1b { z0.h }, p1, [x1]
|
|
; COMMON-NEXT: st1b { z1.h }, p1, [x1, #1, mul vl]
|
|
; COMMON-NEXT: incb x1, all, mul #2
|
|
; COMMON-NEXT: b.ne .LBB1_1
|
|
; COMMON-NEXT: // %bb.2: // %for.exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%mul = shl nuw nsw i64 %vscale, 5
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%src_row_addr = phi ptr [ %src_rows, %entry ], [ %add_ptr_src, %for.body ]
|
|
%dst_row_addr = phi ptr [ %dst_rows, %entry ], [ %add_ptr_dst, %for.body ]
|
|
%idx = phi i64 [ 0, %entry ], [ %inc, %for.body ]
|
|
%2 = load <vscale x 16 x i8>, ptr %src_row_addr
|
|
%3 = getelementptr <vscale x 16 x i8>, ptr %src_row_addr, i64 1
|
|
%4 = load <vscale x 16 x i8>, ptr %3
|
|
%arrayidx2 = getelementptr inbounds i8, ptr %src_row_addr, i64 %stride
|
|
%5 = load <vscale x 16 x i8>, ptr %arrayidx2
|
|
%6 = getelementptr <vscale x 16 x i8>, ptr %arrayidx2, i64 1
|
|
%7 = load <vscale x 16 x i8>, ptr %6
|
|
%8 = add <vscale x 16 x i8> %2, %5
|
|
%9 = add <vscale x 16 x i8> %4, %7
|
|
%10 = bitcast <vscale x 16 x i8> %8 to <vscale x 8 x i16>
|
|
%11 = trunc <vscale x 8 x i16> %10 to <vscale x 8 x i8>
|
|
store <vscale x 8 x i8> %11, ptr %dst_row_addr
|
|
%12 = bitcast <vscale x 16 x i8> %9 to <vscale x 8 x i16>
|
|
%13 = getelementptr <vscale x 8 x i8>, ptr %dst_row_addr, i64 1
|
|
%14 = trunc <vscale x 8 x i16> %12 to <vscale x 8 x i8>
|
|
store <vscale x 8 x i8> %14, ptr %13
|
|
%add_ptr_src = getelementptr inbounds i8, ptr %src_row_addr, i64 %mul
|
|
%add_ptr_dst = getelementptr inbounds i8, ptr %dst_row_addr, i64 %mul
|
|
%inc = add nuw i64 %idx, 1
|
|
%exitcond = icmp eq i64 %inc, %count
|
|
br i1 %exitcond, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
define void @fixed_iv_scalable_offset(ptr %src, ptr %dst, i64 %count) #0 {
|
|
; COMMON-LABEL: fixed_iv_scalable_offset:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: .LBB2_1: // %for.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: ldr z0, [x0]
|
|
; COMMON-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; COMMON-NEXT: subs x2, x2, #4
|
|
; COMMON-NEXT: add x0, x0, #16
|
|
; COMMON-NEXT: add z0.s, z0.s, z1.s
|
|
; COMMON-NEXT: str z0, [x1]
|
|
; COMMON-NEXT: add x1, x1, #16
|
|
; COMMON-NEXT: b.ne .LBB2_1
|
|
; COMMON-NEXT: // %bb.2: // %for.exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%mul = shl nuw nsw i64 %vscale, 4
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %inc, %for.body ]
|
|
%src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%data = load <vscale x 4 x i32>, ptr %src.ptr
|
|
%src.ptr.offset = getelementptr inbounds i32, ptr %src.ptr, i64 %mul
|
|
%data2 = load <vscale x 4 x i32>, ptr %src.ptr.offset
|
|
%add = add <vscale x 4 x i32> %data, %data2
|
|
%dst.ptr = getelementptr i32, ptr %dst, i64 %iv
|
|
store <vscale x 4 x i32> %add, ptr %dst.ptr
|
|
%inc = add nuw i64 %iv, 4
|
|
%exit.cond = icmp eq i64 %inc, %count
|
|
br i1 %exit.cond, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
define void @mixed_offsets_scalable_then_fixed(ptr %src, ptr %dst, i64 %count) #0 {
|
|
; BASE-LABEL: mixed_offsets_scalable_then_fixed:
|
|
; BASE: // %bb.0: // %entry
|
|
; BASE-NEXT: rdvl x8, #4
|
|
; BASE-NEXT: ptrue p0.b
|
|
; BASE-NEXT: orr x8, x8, #0x20
|
|
; BASE-NEXT: .LBB3_1: // %for.body
|
|
; BASE-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; BASE-NEXT: ldr z0, [x0]
|
|
; BASE-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; BASE-NEXT: decw x2
|
|
; BASE-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
|
|
; BASE-NEXT: incb x0
|
|
; BASE-NEXT: add z0.s, z0.s, z1.s
|
|
; BASE-NEXT: add z0.s, z0.s, z2.s
|
|
; BASE-NEXT: str z0, [x1]
|
|
; BASE-NEXT: incb x1
|
|
; BASE-NEXT: cbnz x2, .LBB3_1
|
|
; BASE-NEXT: // %bb.2: // %for.exit
|
|
; BASE-NEXT: ret
|
|
;
|
|
; PREINDEX-LABEL: mixed_offsets_scalable_then_fixed:
|
|
; PREINDEX: // %bb.0: // %entry
|
|
; PREINDEX-NEXT: rdvl x8, #4
|
|
; PREINDEX-NEXT: ptrue p0.b
|
|
; PREINDEX-NEXT: orr x8, x8, #0x20
|
|
; PREINDEX-NEXT: .LBB3_1: // %for.body
|
|
; PREINDEX-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; PREINDEX-NEXT: ldr z0, [x0]
|
|
; PREINDEX-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; PREINDEX-NEXT: decw x2
|
|
; PREINDEX-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
|
|
; PREINDEX-NEXT: incb x0
|
|
; PREINDEX-NEXT: add z0.s, z0.s, z1.s
|
|
; PREINDEX-NEXT: add z0.s, z0.s, z2.s
|
|
; PREINDEX-NEXT: str z0, [x1]
|
|
; PREINDEX-NEXT: incb x1
|
|
; PREINDEX-NEXT: cbnz x2, .LBB3_1
|
|
; PREINDEX-NEXT: // %bb.2: // %for.exit
|
|
; PREINDEX-NEXT: ret
|
|
;
|
|
; POSTINDEX-LABEL: mixed_offsets_scalable_then_fixed:
|
|
; POSTINDEX: // %bb.0: // %entry
|
|
; POSTINDEX-NEXT: ptrue p0.b
|
|
; POSTINDEX-NEXT: rdvl x9, #4
|
|
; POSTINDEX-NEXT: mov x8, xzr
|
|
; POSTINDEX-NEXT: ptrue p1.s
|
|
; POSTINDEX-NEXT: orr x9, x9, #0x20
|
|
; POSTINDEX-NEXT: .LBB3_1: // %for.body
|
|
; POSTINDEX-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; POSTINDEX-NEXT: ldr z0, [x0]
|
|
; POSTINDEX-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; POSTINDEX-NEXT: ld1b { z2.b }, p0/z, [x0, x9]
|
|
; POSTINDEX-NEXT: incb x0
|
|
; POSTINDEX-NEXT: add z0.s, z0.s, z1.s
|
|
; POSTINDEX-NEXT: add z0.s, z0.s, z2.s
|
|
; POSTINDEX-NEXT: st1w { z0.s }, p1, [x1, x8, lsl #2]
|
|
; POSTINDEX-NEXT: incw x8
|
|
; POSTINDEX-NEXT: cmp x2, x8
|
|
; POSTINDEX-NEXT: b.ne .LBB3_1
|
|
; POSTINDEX-NEXT: // %bb.2: // %for.exit
|
|
; POSTINDEX-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%mul = shl nuw nsw i64 %vscale, 4
|
|
%vl = shl nuw nsw i64 %vscale, 2
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %inc, %for.body ]
|
|
%src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%data = load <vscale x 4 x i32>, ptr %src.ptr
|
|
%src.ptr.sc_off = getelementptr inbounds i32, ptr %src.ptr, i64 %mul
|
|
%data2 = load <vscale x 4 x i32>, ptr %src.ptr.sc_off
|
|
%src.ptr.fx_off = getelementptr inbounds i32, ptr %src.ptr.sc_off, i64 8
|
|
%data3 = load <vscale x 4 x i32>, ptr %src.ptr.fx_off
|
|
%add = add <vscale x 4 x i32> %data, %data2
|
|
%add2 = add <vscale x 4 x i32> %add, %data3
|
|
%dst.ptr = getelementptr i32, ptr %dst, i64 %iv
|
|
store <vscale x 4 x i32> %add2, ptr %dst.ptr
|
|
%inc = add nuw i64 %iv, %vl
|
|
%exit.cond = icmp eq i64 %inc, %count
|
|
br i1 %exit.cond, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
define void @mixed_offsets_fixed_then_scalable(ptr %src, ptr %dst, i64 %count) #0 {
|
|
; COMMON-LABEL: mixed_offsets_fixed_then_scalable:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: ptrue p0.s
|
|
; COMMON-NEXT: rdvl x9, #4
|
|
; COMMON-NEXT: mov x8, xzr
|
|
; COMMON-NEXT: ptrue p1.b
|
|
; COMMON-NEXT: orr x9, x9, #0x20
|
|
; COMMON-NEXT: mov x10, #8 // =0x8
|
|
; COMMON-NEXT: .LBB4_1: // %for.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: ldr z0, [x0]
|
|
; COMMON-NEXT: ld1w { z1.s }, p0/z, [x0, x10, lsl #2]
|
|
; COMMON-NEXT: ld1b { z2.b }, p1/z, [x0, x9]
|
|
; COMMON-NEXT: incb x0
|
|
; COMMON-NEXT: add z0.s, z0.s, z1.s
|
|
; COMMON-NEXT: add z0.s, z0.s, z2.s
|
|
; COMMON-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
|
|
; COMMON-NEXT: incw x8
|
|
; COMMON-NEXT: cmp x2, x8
|
|
; COMMON-NEXT: b.ne .LBB4_1
|
|
; COMMON-NEXT: // %bb.2: // %for.exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%mul = shl nuw nsw i64 %vscale, 4
|
|
%vl = shl nuw nsw i64 %vscale, 2
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %inc, %for.body ]
|
|
%src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%data = load <vscale x 4 x i32>, ptr %src.ptr
|
|
%src.ptr.fx_off = getelementptr inbounds i32, ptr %src.ptr, i64 8
|
|
%data2 = load <vscale x 4 x i32>, ptr %src.ptr.fx_off
|
|
%src.ptr.sc_off = getelementptr inbounds i32, ptr %src.ptr.fx_off, i64 %mul
|
|
%data3 = load <vscale x 4 x i32>, ptr %src.ptr.sc_off
|
|
%add = add <vscale x 4 x i32> %data, %data2
|
|
%add2 = add <vscale x 4 x i32> %add, %data3
|
|
%dst.ptr = getelementptr i32, ptr %dst, i64 %iv
|
|
store <vscale x 4 x i32> %add2, ptr %dst.ptr
|
|
%inc = add nuw i64 %iv, %vl
|
|
%exit.cond = icmp eq i64 %inc, %count
|
|
br i1 %exit.cond, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
;; FIXME: There's an opportunity here (that we currently miss) to define the phi
|
|
;; on the middle access, and have negative and positive scalable immediates.
|
|
;;
|
|
;; Currently we generate a scalable offset for the load in range of the base,
|
|
;; and a register to store the offset for the access that's out of range of the
|
|
;; base (but in range of the other).
|
|
;;
|
|
define void @three_access_wide_gap(ptr %src, ptr %dst, i64 %count) #0 {
|
|
; BASE-LABEL: three_access_wide_gap:
|
|
; BASE: // %bb.0: // %entry
|
|
; BASE-NEXT: .LBB5_1: // %for.body
|
|
; BASE-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; BASE-NEXT: ldr z0, [x0]
|
|
; BASE-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; BASE-NEXT: decw x2
|
|
; BASE-NEXT: ldr z2, [x0, #8, mul vl]
|
|
; BASE-NEXT: incb x0
|
|
; BASE-NEXT: add z0.s, z0.s, z1.s
|
|
; BASE-NEXT: add z0.s, z0.s, z2.s
|
|
; BASE-NEXT: str z0, [x1]
|
|
; BASE-NEXT: incb x1
|
|
; BASE-NEXT: cbnz x2, .LBB5_1
|
|
; BASE-NEXT: // %bb.2: // %for.exit
|
|
; BASE-NEXT: ret
|
|
;
|
|
; PREINDEX-LABEL: three_access_wide_gap:
|
|
; PREINDEX: // %bb.0: // %entry
|
|
; PREINDEX-NEXT: .LBB5_1: // %for.body
|
|
; PREINDEX-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; PREINDEX-NEXT: ldr z0, [x0]
|
|
; PREINDEX-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; PREINDEX-NEXT: decw x2
|
|
; PREINDEX-NEXT: ldr z2, [x0, #8, mul vl]
|
|
; PREINDEX-NEXT: incb x0
|
|
; PREINDEX-NEXT: add z0.s, z0.s, z1.s
|
|
; PREINDEX-NEXT: add z0.s, z0.s, z2.s
|
|
; PREINDEX-NEXT: str z0, [x1]
|
|
; PREINDEX-NEXT: incb x1
|
|
; PREINDEX-NEXT: cbnz x2, .LBB5_1
|
|
; PREINDEX-NEXT: // %bb.2: // %for.exit
|
|
; PREINDEX-NEXT: ret
|
|
;
|
|
; POSTINDEX-LABEL: three_access_wide_gap:
|
|
; POSTINDEX: // %bb.0: // %entry
|
|
; POSTINDEX-NEXT: ptrue p0.s
|
|
; POSTINDEX-NEXT: mov x8, xzr
|
|
; POSTINDEX-NEXT: .LBB5_1: // %for.body
|
|
; POSTINDEX-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; POSTINDEX-NEXT: ldr z0, [x0]
|
|
; POSTINDEX-NEXT: ldr z1, [x0, #4, mul vl]
|
|
; POSTINDEX-NEXT: ldr z2, [x0, #8, mul vl]
|
|
; POSTINDEX-NEXT: incb x0
|
|
; POSTINDEX-NEXT: add z0.s, z0.s, z1.s
|
|
; POSTINDEX-NEXT: add z0.s, z0.s, z2.s
|
|
; POSTINDEX-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
|
|
; POSTINDEX-NEXT: incw x8
|
|
; POSTINDEX-NEXT: cmp x2, x8
|
|
; POSTINDEX-NEXT: b.ne .LBB5_1
|
|
; POSTINDEX-NEXT: // %bb.2: // %for.exit
|
|
; POSTINDEX-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%mul = mul nuw nsw i64 %vscale, 16
|
|
%mul2 = mul nuw nsw i64 %vscale, 16
|
|
%vl = mul nuw nsw i64 %vscale, 4
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %inc, %for.body ]
|
|
%src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%data = load <vscale x 4 x i32>, ptr %src.ptr
|
|
%src.ptr.sc_off = getelementptr inbounds i32, ptr %src.ptr, i64 %mul
|
|
%data2 = load <vscale x 4 x i32>, ptr %src.ptr.sc_off
|
|
%src.ptr.sc_off2 = getelementptr inbounds i32, ptr %src.ptr.sc_off, i64 %mul2
|
|
%data3 = load <vscale x 4 x i32>, ptr %src.ptr.sc_off2
|
|
%add = add <vscale x 4 x i32> %data, %data2
|
|
%add2 = add <vscale x 4 x i32> %add, %data3
|
|
%dst.ptr = getelementptr i32, ptr %dst, i64 %iv
|
|
store <vscale x 4 x i32> %add2, ptr %dst.ptr
|
|
%inc = add nuw i64 %iv, %vl
|
|
%exit.cond = icmp eq i64 %inc, %count
|
|
br i1 %exit.cond, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
;; Here are two writes that should be `16 * vscale * vscale` apart, so MUL VL
|
|
;; addressing cannot be used to offset the second write, as for example,
|
|
;; `#4, mul vl` would only be an offset of `16 * vscale` (dropping a vscale).
|
|
define void @vscale_squared_offset(ptr %alloc) #0 {
|
|
; COMMON-LABEL: vscale_squared_offset:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: rdvl x9, #1
|
|
; COMMON-NEXT: rdvl x10, #4
|
|
; COMMON-NEXT: fmov z0.s, #4.00000000
|
|
; COMMON-NEXT: lsr x9, x9, #4
|
|
; COMMON-NEXT: fmov z1.s, #8.00000000
|
|
; COMMON-NEXT: mov x8, xzr
|
|
; COMMON-NEXT: ptrue p0.s, vl1
|
|
; COMMON-NEXT: umull x9, w9, w10
|
|
; COMMON-NEXT: cntw x10
|
|
; COMMON-NEXT: cmp x8, x10
|
|
; COMMON-NEXT: b.ge .LBB6_2
|
|
; COMMON-NEXT: .LBB6_1: // %for.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: add x11, x0, x9
|
|
; COMMON-NEXT: st1w { z0.s }, p0, [x0]
|
|
; COMMON-NEXT: incb x0
|
|
; COMMON-NEXT: st1w { z1.s }, p0, [x11]
|
|
; COMMON-NEXT: add x8, x8, #1
|
|
; COMMON-NEXT: cmp x8, x10
|
|
; COMMON-NEXT: b.lt .LBB6_1
|
|
; COMMON-NEXT: .LBB6_2: // %for.exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%vscale = call i64 @llvm.vscale.i64()
|
|
%c4_vscale = mul i64 %vscale, 4
|
|
br label %for.check
|
|
for.check:
|
|
%i = phi i64 [ %next_i, %for.body ], [ 0, %entry ]
|
|
%is_lt = icmp slt i64 %i, %c4_vscale
|
|
br i1 %is_lt, label %for.body, label %for.exit
|
|
for.body:
|
|
%mask = call <vscale x 4 x i1> @llvm.aarch64.sve.whilelt.nxv4i1.i64(i64 0, i64 1)
|
|
%upper_offset = mul i64 %i, %c4_vscale
|
|
%upper_ptr = getelementptr float, ptr %alloc, i64 %upper_offset
|
|
call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 4.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), ptr %upper_ptr, i32 4, <vscale x 4 x i1> %mask)
|
|
%lower_i = add i64 %i, %c4_vscale
|
|
%lower_offset = mul i64 %lower_i, %c4_vscale
|
|
%lower_ptr = getelementptr float, ptr %alloc, i64 %lower_offset
|
|
call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 8.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), ptr %lower_ptr, i32 4, <vscale x 4 x i1> %mask)
|
|
%next_i = add i64 %i, 1
|
|
br label %for.check
|
|
for.exit:
|
|
ret void
|
|
}
|
|
|
|
; In this test the accesses are offset by the vector length (i.e, #1, mul vl apart),
|
|
; but include a non-trival SCEV expression for the base address.
|
|
; The SCEV expressions for the accesses are:
|
|
; %gep.part0 =
|
|
; {(324 + (4 * (sext i32 (3 + (-1 * %3)<nsw> + %0) to i64))<nsw> + %ptr),+,(32 * vscale)<nuw><nsw>}<%vector.body>
|
|
; %gep.part1 =
|
|
; {(324 + (4 * (sext i32 (3 + (-1 * %3)<nsw> + %0) to i64))<nsw> + (16 * vscale)<nuw><nsw> + %ptr),+,(32 * vscale)<nuw><nsw>}
|
|
; This test checks that LSR can extract the offset of `(16 * vscale)` from the
|
|
; expression for %gep.part1 (and use mul vl addressing from a common base for
|
|
; both loads and stores).
|
|
define void @vscale_complex_base_address(ptr %ptr, i32 %0, i64 %n) local_unnamed_addr #0 {
|
|
; COMMON-LABEL: vscale_complex_base_address:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: mov w9, #21846 // =0x5556
|
|
; COMMON-NEXT: sub w8, w1, #1
|
|
; COMMON-NEXT: movk w9, #21845, lsl #16
|
|
; COMMON-NEXT: smull x8, w8, w9
|
|
; COMMON-NEXT: lsr x9, x8, #32
|
|
; COMMON-NEXT: add x8, x9, x8, lsr #63
|
|
; COMMON-NEXT: add w8, w8, w8, lsl #1
|
|
; COMMON-NEXT: add w8, w8, #4
|
|
; COMMON-NEXT: add x8, x0, w8, sxtw #2
|
|
; COMMON-NEXT: add x8, x8, #324
|
|
; COMMON-NEXT: .LBB7_1: // %vector.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: ldr z0, [x8]
|
|
; COMMON-NEXT: ldr z1, [x8, #1, mul vl]
|
|
; COMMON-NEXT: dech x2
|
|
; COMMON-NEXT: sub z0.s, z0.s, #10 // =0xa
|
|
; COMMON-NEXT: sub z1.s, z1.s, #10 // =0xa
|
|
; COMMON-NEXT: str z0, [x8]
|
|
; COMMON-NEXT: str z1, [x8, #1, mul vl]
|
|
; COMMON-NEXT: incb x8, all, mul #2
|
|
; COMMON-NEXT: cbnz x2, .LBB7_1
|
|
; COMMON-NEXT: // %bb.2: // %exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%1 = add i32 %0, 3
|
|
%2 = add i32 %0, -1
|
|
%3 = srem i32 %2, 3
|
|
%4 = sub i32 %1, %3
|
|
%5 = sext i32 %4 to i64
|
|
%ptr.offset = getelementptr inbounds nuw i8, ptr %ptr, i64 324
|
|
%complex.base = getelementptr [4 x i8], ptr %ptr.offset, i64 %5
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%VFxUF = shl nuw i64 %vscale, 3
|
|
%off = shl i64 %vscale, 4
|
|
br label %vector.body
|
|
|
|
vector.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %vector.body ]
|
|
%gep.part0 = getelementptr [4 x i8], ptr %complex.base, i64 %iv
|
|
%gep.part1 = getelementptr i8, ptr %gep.part0, i64 %off
|
|
%data = load <vscale x 4 x i32>, ptr %gep.part0, align 4
|
|
%data2 = load <vscale x 4 x i32>, ptr %gep.part1, align 4
|
|
%add = add <vscale x 4 x i32> %data, splat (i32 -10)
|
|
%add2 = add <vscale x 4 x i32> %data2, splat (i32 -10)
|
|
store <vscale x 4 x i32> %add, ptr %gep.part0, align 4
|
|
store <vscale x 4 x i32> %add2, ptr %gep.part1, align 4
|
|
%iv.next = add nuw i64 %iv, %VFxUF
|
|
%exit.cond = icmp eq i64 %iv.next, %n
|
|
br i1 %exit.cond, label %exit, label %vector.body
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
; Tests a simple memcpy-like routine where source and destination accesses are
|
|
; offset by 8 bytes. That is the SCEV expression for any load/store is:
|
|
; {(8 + ((16 * offset) * vscale) + %dst)<nuw>,+,(64 * vscale)}
|
|
; where `offset` is 0, 1, 2, or 3. This should use be able to use `mul vl`
|
|
; addressing (with the offset of 8 moved out of the loop).
|
|
define void @offset_memcpy(ptr %src, ptr %dst, i64 %n.vec) "target-cpu"="neoverse-v2" nounwind {
|
|
; COMMON-LABEL: offset_memcpy:
|
|
; COMMON: // %bb.0: // %entry
|
|
; COMMON-NEXT: cnth x8
|
|
; COMMON-NEXT: add x9, x1, #8
|
|
; COMMON-NEXT: add x10, x0, #8
|
|
; COMMON-NEXT: .p2align 5, , 16
|
|
; COMMON-NEXT: .LBB8_1: // %vector.body
|
|
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; COMMON-NEXT: ldr z0, [x10]
|
|
; COMMON-NEXT: ldr z1, [x10, #1, mul vl]
|
|
; COMMON-NEXT: ldr z2, [x10, #2, mul vl]
|
|
; COMMON-NEXT: subs x2, x2, x8
|
|
; COMMON-NEXT: ldr z3, [x10, #3, mul vl]
|
|
; COMMON-NEXT: incb x10, all, mul #4
|
|
; COMMON-NEXT: str z0, [x9]
|
|
; COMMON-NEXT: str z1, [x9, #1, mul vl]
|
|
; COMMON-NEXT: str z2, [x9, #2, mul vl]
|
|
; COMMON-NEXT: str z3, [x9, #3, mul vl]
|
|
; COMMON-NEXT: incb x9, all, mul #4
|
|
; COMMON-NEXT: b.ne .LBB8_1
|
|
; COMMON-NEXT: // %bb.2: // %exit
|
|
; COMMON-NEXT: ret
|
|
entry:
|
|
%7 = call i64 @llvm.vscale.i64()
|
|
%8 = shl nuw i64 %7, 1
|
|
%9 = shl nuw i64 %8, 2
|
|
br label %vector.body
|
|
|
|
vector.body:
|
|
%index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
|
|
%11 = add i64 1, %index
|
|
%12 = getelementptr inbounds nuw [8 x i8], ptr %src, i64 %11
|
|
%13 = shl nuw nsw i64 %8, 1
|
|
%14 = mul nuw nsw i64 %8, 3
|
|
%15 = getelementptr inbounds nuw ptr, ptr %12, i64 %8
|
|
%16 = getelementptr inbounds nuw ptr, ptr %12, i64 %13
|
|
%17 = getelementptr inbounds nuw ptr, ptr %12, i64 %14
|
|
%wide.load = load <vscale x 2 x ptr>, ptr %12, align 8
|
|
%wide.load3 = load <vscale x 2 x ptr>, ptr %15, align 8
|
|
%wide.load4 = load <vscale x 2 x ptr>, ptr %16, align 8
|
|
%wide.load5 = load <vscale x 2 x ptr>, ptr %17, align 8
|
|
%18 = getelementptr inbounds nuw [8 x i8], ptr %dst, i64 %11
|
|
%19 = getelementptr inbounds nuw ptr, ptr %18, i64 %8
|
|
%20 = getelementptr inbounds nuw ptr, ptr %18, i64 %13
|
|
%21 = getelementptr inbounds nuw ptr, ptr %18, i64 %14
|
|
store <vscale x 2 x ptr> %wide.load, ptr %18, align 8
|
|
store <vscale x 2 x ptr> %wide.load3, ptr %19, align 8
|
|
store <vscale x 2 x ptr> %wide.load4, ptr %20, align 8
|
|
store <vscale x 2 x ptr> %wide.load5, ptr %21, align 8
|
|
%index.next = add nuw i64 %index, %9
|
|
%22 = icmp eq i64 %index.next, %n.vec
|
|
br i1 %22, label %exit, label %vector.body
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
|