derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
532 lines
18 KiB
LLVM
532 lines
18 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 5
|
|
; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=arm64e-apple-darwin -o - | FileCheck %s
|
|
|
|
|
|
define i16 @sabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: sabal_i8_to_i16_accumulation:
|
|
; CHECK: sabdl.8h v1, v1, v3
|
|
; CHECK: sabdl.8h v0, v0, v2
|
|
; CHECK: sabdl.8h v2, v3, v5
|
|
; CHECK: sabal.8h v1, v4, v6
|
|
; CHECK: sabal.8h v0, v7, v17
|
|
; CHECK: sabal.8h v2, v16, v18
|
|
; CHECK: sabal.8h v1, v19, v21
|
|
; CHECK: sabal.8h v0, v20, v22
|
|
; CHECK: add.8h v1, v2, v1
|
|
; CHECK: add.8h v0, v1, v0
|
|
; CHECK: addv.8h h0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <8 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i8>, ptr %ptr2_i, align 1
|
|
%vabd = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %a, <8 x i8> %b)
|
|
%vabd_ext = zext <8 x i8> %vabd to <8 x i16>
|
|
%acc_next = add <8 x i16> %vabd_ext, %acc_phi
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
; Declare the signed absolute difference intrinsic
|
|
declare <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8>, <8 x i8>)
|
|
declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
|
|
|
|
|
|
define i32 @sabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: sabal_i16_to_i32_accumulation:
|
|
; CHECK: sabdl.4s v1, v1, v3
|
|
; CHECK: sabdl.4s v0, v0, v2
|
|
; CHECK: sabdl.4s v2, v3, v5
|
|
; CHECK: sabal.4s v1, v4, v6
|
|
; CHECK: sabal.4s v0, v7, v17
|
|
; CHECK: sabal.4s v2, v16, v18
|
|
; CHECK: sabal.4s v1, v19, v21
|
|
; CHECK: sabal.4s v0, v20, v22
|
|
; CHECK: add.4s v1, v2, v1
|
|
; CHECK: add.4s v0, v1, v0
|
|
; CHECK: addv.4s s0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
|
|
%a = load <4 x i16>, ptr %ptr1_i, align 1
|
|
%b = load <4 x i16>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %a, <4 x i16> %b)
|
|
%vmov = zext <4 x i16> %vabd to <4 x i32>
|
|
%acc_next = add <4 x i32> %vmov, %acc_phi
|
|
%next_i = add i32 %i, 4
|
|
%cmp = icmp slt i32 %next_i, 32
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)
|
|
ret i32 %reduce
|
|
}
|
|
|
|
declare <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16>, <4 x i16>)
|
|
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
|
|
|
|
define i16 @uabal2_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uabal2_accumulation:
|
|
; CHECK: uabdl2.8h v4, v1, v3
|
|
; CHECK: uabdl.8h v1, v1, v3
|
|
; CHECK: uabdl2.8h v24, v0, v2
|
|
; CHECK: uabdl2.8h v25, v3, v6
|
|
; CHECK: uabal2.8h v4, v5, v7
|
|
; CHECK: uabal2.8h v24, v16, v18
|
|
; CHECK: uabal2.8h v25, v17, v19
|
|
; CHECK: uabal2.8h v4, v20, v22
|
|
; CHECK: uabal2.8h v24, v21, v23
|
|
; CHECK: add.8h v4, v25, v4
|
|
; CHECK: add.8h v4, v4, v24
|
|
; CHECK: uabdl.8h v0, v0, v2
|
|
; CHECK: uabdl.8h v2, v3, v6
|
|
; CHECK: uabal.8h v1, v5, v7
|
|
; CHECK: uabal.8h v0, v16, v18
|
|
; CHECK: uabal.8h v2, v17, v19
|
|
; CHECK: uabal.8h v1, v20, v22
|
|
; CHECK: uabal.8h v0, v21, v23
|
|
; CHECK: add.8h v1, v2, v1
|
|
; CHECK: add.8h v0, v1, v0
|
|
; CHECK: add.8h v0, v4, v0
|
|
; CHECK: addv.8h h0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi_hi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]
|
|
%acc_phi_lo = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next_lo, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <16 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <16 x i8>, ptr %ptr2_i, align 1
|
|
%a_hi = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
|
%b_hi = shufflevector <16 x i8> %b, <16 x i8> zeroinitializer, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
|
%a_lo = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
%b_lo = shufflevector <16 x i8> %b, <16 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
%vabd_hi = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a_hi, <8 x i8> %b_hi)
|
|
%vabd_lo = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a_lo, <8 x i8> %b_lo)
|
|
%vmov_hi = zext <8 x i8> %vabd_hi to <8 x i16>
|
|
%vmov_lo = zext <8 x i8> %vabd_lo to <8 x i16>
|
|
%acc_next_hi = add <8 x i16> %vmov_hi, %acc_phi_hi
|
|
%acc_next_lo = add <8 x i16> %vmov_lo, %acc_phi_lo
|
|
%next_i = add i32 %i, 16
|
|
%cmp = icmp slt i32 %next_i, 128
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%hi_plus_lo = add <8 x i16> %acc_next_hi, %acc_next_lo
|
|
%reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %hi_plus_lo)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
define i32 @uaba_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uaba_accumulation:
|
|
; CHECK: uabd.4s v0, v0, v2
|
|
; CHECK: uabd.4s v1, v1, v3
|
|
; CHECK: uabd.4s v2, v2, v5
|
|
; CHECK: uaba.4s v0, v4, v6
|
|
; CHECK: uaba.4s v1, v7, v17
|
|
; CHECK: uaba.4s v2, v16, v18
|
|
; CHECK: uaba.4s v0, v19, v21
|
|
; CHECK: uaba.4s v1, v20, v22
|
|
; CHECK: add.4s v0, v2, v0
|
|
; CHECK: add.4s v0, v0, v1
|
|
; CHECK: addv.4s s0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
|
|
%a = load <4 x i32>, ptr %ptr1_i, align 1
|
|
%b = load <4 x i32>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %a, <4 x i32> %b)
|
|
%acc_next = add <4 x i32> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 4
|
|
%cmp = icmp slt i32 %next_i, 32
|
|
br i1 %cmp, label %loop, label %exit
|
|
exit:
|
|
|
|
%reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)
|
|
ret i32 %reduce
|
|
}
|
|
|
|
declare <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
|
|
|
|
define i32 @saba_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: saba_accumulation:
|
|
; CHECK: sabd.4s v0, v0, v2
|
|
; CHECK: sabd.4s v1, v1, v3
|
|
; CHECK: sabd.4s v2, v2, v5
|
|
; CHECK: saba.4s v0, v4, v6
|
|
; CHECK: saba.4s v1, v7, v17
|
|
; CHECK: saba.4s v2, v16, v18
|
|
; CHECK: saba.4s v0, v19, v21
|
|
; CHECK: saba.4s v1, v20, v22
|
|
; CHECK: add.4s v0, v2, v0
|
|
; CHECK: add.4s v0, v0, v1
|
|
; CHECK: addv.4s s0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
; Load values from ptr1 and ptr2
|
|
%ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
|
|
%a = load <4 x i32>, ptr %ptr1_i, align 1
|
|
%b = load <4 x i32>, ptr %ptr2_i, align 1
|
|
; Perform the intrinsic operation
|
|
%vabd = tail call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %a, <4 x i32> %b)
|
|
%acc_next = add <4 x i32> %acc_phi, %vabd
|
|
; Increment loop counter and check the bound
|
|
%next_i = add i32 %i, 4
|
|
%cmp = icmp slt i32 %next_i, 32
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)
|
|
ret i32 %reduce
|
|
}
|
|
|
|
declare <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
|
|
|
|
define i32 @uaba_v2i32_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uaba_v2i32_accumulation:
|
|
; CHECK: uabd.2s v0, v0, v2
|
|
; CHECK: uabd.2s v1, v1, v3
|
|
; CHECK: uabd.2s v2, v2, v5
|
|
; CHECK: uaba.2s v0, v4, v6
|
|
; CHECK: uaba.2s v1, v7, v17
|
|
; CHECK: uaba.2s v2, v16, v18
|
|
; CHECK: uaba.2s v0, v19, v21
|
|
; CHECK: uaba.2s v1, v20, v22
|
|
; CHECK: add.2s v0, v2, v0
|
|
; CHECK: add.2s v0, v0, v1
|
|
; CHECK: addp.2s v0, v0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <2 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i32, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i32, ptr %ptr2, i32 %i
|
|
%a = load <2 x i32>, ptr %ptr1_i, align 1
|
|
%b = load <2 x i32>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %a, <2 x i32> %b)
|
|
%acc_next = add <2 x i32> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 2
|
|
%cmp = icmp slt i32 %next_i, 16
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %acc_next)
|
|
ret i32 %reduce
|
|
}
|
|
|
|
define i8 @uaba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uaba_v8i8_accumulation:
|
|
; CHECK: uabd.8b v0, v0, v2
|
|
; CHECK: uabd.8b v1, v1, v3
|
|
; CHECK: uabd.8b v2, v2, v5
|
|
; CHECK: uaba.8b v0, v4, v6
|
|
; CHECK: uaba.8b v1, v7, v17
|
|
; CHECK: uaba.8b v2, v16, v18
|
|
; CHECK: uaba.8b v0, v19, v21
|
|
; CHECK: uaba.8b v1, v20, v22
|
|
; CHECK: add.8b v0, v2, v0
|
|
; CHECK: add.8b v0, v0, v1
|
|
; CHECK: addv.8b b0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <8 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i8>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a, <8 x i8> %b)
|
|
%acc_next = add <8 x i8> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %acc_next)
|
|
ret i8 %reduce
|
|
}
|
|
|
|
define i8 @uaba_v16i8_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uaba_v16i8_accumulation:
|
|
; CHECK: uabd.16b v0, v0, v2
|
|
; CHECK: uabd.16b v1, v1, v3
|
|
; CHECK: uabd.16b v2, v2, v5
|
|
; CHECK: uaba.16b v0, v4, v6
|
|
; CHECK: uaba.16b v1, v7, v17
|
|
; CHECK: uaba.16b v2, v16, v18
|
|
; CHECK: uaba.16b v0, v19, v21
|
|
; CHECK: uaba.16b v1, v20, v22
|
|
; CHECK: add.16b v0, v2, v0
|
|
; CHECK: add.16b v0, v0, v1
|
|
; CHECK: addv.16b b0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <16 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <16 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <16 x i8>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %a, <16 x i8> %b)
|
|
%acc_next = add <16 x i8> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 16
|
|
%cmp = icmp slt i32 %next_i, 128
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %acc_next)
|
|
ret i8 %reduce
|
|
}
|
|
|
|
define i16 @uaba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uaba_v8i16_accumulation:
|
|
; CHECK: uabd.8h v0, v0, v2
|
|
; CHECK: uabd.8h v1, v1, v3
|
|
; CHECK: uabd.8h v2, v2, v5
|
|
; CHECK: uaba.8h v0, v4, v6
|
|
; CHECK: uaba.8h v1, v7, v17
|
|
; CHECK: uaba.8h v2, v16, v18
|
|
; CHECK: uaba.8h v0, v19, v21
|
|
; CHECK: uaba.8h v1, v20, v22
|
|
; CHECK: add.8h v0, v2, v0
|
|
; CHECK: add.8h v0, v0, v1
|
|
; CHECK: addv.8h h0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
|
|
%a = load <8 x i16>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i16>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %a, <8 x i16> %b)
|
|
%acc_next = add <8 x i16> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
define i8 @saba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: saba_v8i8_accumulation:
|
|
; CHECK: sabd.8b v0, v0, v2
|
|
; CHECK: sabd.8b v1, v1, v3
|
|
; CHECK: sabd.8b v2, v2, v5
|
|
; CHECK: saba.8b v0, v4, v6
|
|
; CHECK: saba.8b v1, v7, v17
|
|
; CHECK: saba.8b v2, v16, v18
|
|
; CHECK: saba.8b v0, v19, v21
|
|
; CHECK: saba.8b v1, v20, v22
|
|
; CHECK: add.8b v0, v2, v0
|
|
; CHECK: add.8b v0, v0, v1
|
|
; CHECK: addv.8b b0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <8 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i8>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %a, <8 x i8> %b)
|
|
%acc_next = add <8 x i8> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %acc_next)
|
|
ret i8 %reduce
|
|
}
|
|
|
|
define i16 @saba_v4i16_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: saba_v4i16_accumulation:
|
|
; CHECK: sabd.4h v0, v0, v2
|
|
; CHECK: sabd.4h v1, v1, v3
|
|
; CHECK: sabd.4h v2, v2, v5
|
|
; CHECK: saba.4h v0, v4, v6
|
|
; CHECK: saba.4h v1, v7, v17
|
|
; CHECK: saba.4h v2, v16, v18
|
|
; CHECK: saba.4h v0, v19, v21
|
|
; CHECK: saba.4h v1, v20, v22
|
|
; CHECK: add.4h v0, v2, v0
|
|
; CHECK: add.4h v0, v0, v1
|
|
; CHECK: addv.4h h0, v0
|
|
entry:
|
|
br label %loop
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <4 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
|
|
%a = load <4 x i16>, ptr %ptr1_i, align 1
|
|
%b = load <4 x i16>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %a, <4 x i16> %b)
|
|
%acc_next = add <4 x i16> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 4
|
|
%cmp = icmp slt i32 %next_i, 32
|
|
br i1 %cmp, label %loop, label %exit
|
|
exit:
|
|
%reduce = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %acc_next)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
define i16 @saba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: saba_v8i16_accumulation:
|
|
; CHECK: sabd.8h v0, v0, v2
|
|
; CHECK: sabd.8h v1, v1, v3
|
|
; CHECK: sabd.8h v2, v2, v5
|
|
; CHECK: saba.8h v0, v4, v6
|
|
; CHECK: saba.8h v1, v7, v17
|
|
; CHECK: saba.8h v2, v16, v18
|
|
; CHECK: saba.8h v0, v19, v21
|
|
; CHECK: saba.8h v1, v20, v22
|
|
; CHECK: add.8h v0, v2, v0
|
|
; CHECK: add.8h v0, v0, v1
|
|
; CHECK: addv.8h h0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
|
|
%a = load <8 x i16>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i16>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16> %a, <8 x i16> %b)
|
|
%acc_next = add <8 x i16> %acc_phi, %vabd
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
define i16 @uabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uabal_i8_to_i16_accumulation:
|
|
; CHECK: uabdl.8h v1, v1, v3
|
|
; CHECK: uabdl.8h v0, v0, v2
|
|
; CHECK: uabdl.8h v2, v3, v5
|
|
; CHECK: uabal.8h v1, v4, v6
|
|
; CHECK: uabal.8h v0, v7, v17
|
|
; CHECK: uabal.8h v2, v16, v18
|
|
; CHECK: uabal.8h v1, v19, v21
|
|
; CHECK: uabal.8h v0, v20, v22
|
|
; CHECK: add.8h v1, v2, v1
|
|
; CHECK: add.8h v0, v1, v0
|
|
; CHECK: addv.8h h0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i8, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i8, ptr %ptr2, i32 %i
|
|
%a = load <8 x i8>, ptr %ptr1_i, align 1
|
|
%b = load <8 x i8>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a, <8 x i8> %b)
|
|
%vmov = zext <8 x i8> %vabd to <8 x i16>
|
|
%acc_next = add <8 x i16> %vmov, %acc_phi
|
|
%next_i = add i32 %i, 8
|
|
%cmp = icmp slt i32 %next_i, 64
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)
|
|
ret i16 %reduce
|
|
}
|
|
|
|
define i32 @uabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
|
|
; CHECK-LABEL: uabal_i16_to_i32_accumulation:
|
|
; CHECK: uabdl.4s v1, v1, v3
|
|
; CHECK: uabdl.4s v0, v0, v2
|
|
; CHECK: uabdl.4s v2, v3, v5
|
|
; CHECK: uabal.4s v1, v4, v6
|
|
; CHECK: uabal.4s v0, v7, v17
|
|
; CHECK: uabal.4s v2, v16, v18
|
|
; CHECK: uabal.4s v1, v19, v21
|
|
; CHECK: uabal.4s v0, v20, v22
|
|
; CHECK: add.4s v1, v2, v1
|
|
; CHECK: add.4s v0, v1, v0
|
|
; CHECK: addv.4s s0, v0
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
|
|
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
|
|
%acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
|
|
%ptr1_i = getelementptr i16, ptr %ptr1, i32 %i
|
|
%ptr2_i = getelementptr i16, ptr %ptr2, i32 %i
|
|
%a = load <4 x i16>, ptr %ptr1_i, align 1
|
|
%b = load <4 x i16>, ptr %ptr2_i, align 1
|
|
%vabd = tail call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %a, <4 x i16> %b)
|
|
%vmov = zext <4 x i16> %vabd to <4 x i32>
|
|
%acc_next = add <4 x i32> %vmov, %acc_phi
|
|
%next_i = add i32 %i, 4
|
|
%cmp = icmp slt i32 %next_i, 32
|
|
br i1 %cmp, label %loop, label %exit
|
|
|
|
exit:
|
|
%reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)
|
|
ret i32 %reduce
|
|
}
|