derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
550 lines
24 KiB
LLVM
550 lines
24 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
|
|
; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s -o - | FileCheck %s
|
|
|
|
define void @histogram_i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: mov z3.d, x0
|
|
; CHECK-NEXT: ld1d { z2.d }, p0/z, [z0.d]
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
|
|
; CHECK-NEXT: st1d { z1.d }, p0, [z0.d]
|
|
; CHECK-NEXT: ret
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
;; FIXME: We maybe need some dagcombines here? We're multiplying the output of the histcnt
|
|
;; by 1, so we should be able to remove that and directly add the histcnt to the
|
|
;; current bucket data.
|
|
define void @histogram_i32_literal(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i32_literal:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: ret
|
|
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i32_literal_noscale(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i32_literal_noscale:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, sxtw]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, sxtw]
|
|
; CHECK-NEXT: ret
|
|
|
|
%buckets = getelementptr i8, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i32_promote(ptr %base, <vscale x 2 x i64> %indices, <vscale x 2 x i1> %mask, i32 %inc) #0 {
|
|
; CHECK-LABEL: histogram_i32_promote:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
|
|
; CHECK-NEXT: mov z3.d, x1
|
|
; CHECK-NEXT: ld1w { z2.d }, p0/z, [x0, z0.d, lsl #2]
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
|
|
; CHECK-NEXT: st1w { z1.d }, p0, [x0, z0.d, lsl #2]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 2 x i64> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> %buckets, i32 %inc, <vscale x 2 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i16 %inc) #0 {
|
|
; CHECK-LABEL: histogram_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, w1
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1h { z1.s }, p0, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %buckets, i16 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i8(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i8 %inc) #0 {
|
|
; CHECK-LABEL: histogram_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, w1
|
|
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x0, z0.s, sxtw]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1b { z1.s }, p0, [x0, z0.s, sxtw]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i8, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i8(<vscale x 4 x ptr> %buckets, i8 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_2_lane(ptr %base, <vscale x 2 x i64> %indices, <vscale x 2 x i1> %mask, i16 %inc) #0 {
|
|
; CHECK-LABEL: histogram_i16_2_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
|
|
; CHECK-NEXT: mov z3.d, x1
|
|
; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, z0.d, lsl #1]
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
|
|
; CHECK-NEXT: st1h { z1.d }, p0, [x0, z0.d, lsl #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 2 x i64> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i16(<vscale x 2 x ptr> %buckets, i16 %inc, <vscale x 2 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i8_2_lane(ptr %base, <vscale x 2 x i64> %indices, <vscale x 2 x i1> %mask, i8 %inc) #0 {
|
|
; CHECK-LABEL: histogram_i8_2_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1
|
|
; CHECK-NEXT: mov z3.d, x1
|
|
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, z0.d]
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
|
|
; CHECK-NEXT: st1b { z1.d }, p0, [x0, z0.d]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i8, ptr %base, <vscale x 2 x i64> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i8(<vscale x 2 x ptr> %buckets, i8 %inc, <vscale x 2 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_literal_1(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i16_literal_1:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1h { z1.s }, p0, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %buckets, i16 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_literal_2(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i16_literal_2:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #2 // =0x2
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1h { z1.s }, p0, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %buckets, i16 2, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_literal_3(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i16_literal_3:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #3 // =0x3
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1h { z1.s }, p0, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 4 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %buckets, i16 3, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i64_4_lane(<vscale x 4 x ptr> %buckets, i64 %inc, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i64_4_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.d, x0
|
|
; CHECK-NEXT: ptrue p2.d
|
|
; CHECK-NEXT: histcnt z2.d, p1/z, z0.d, z0.d
|
|
; CHECK-NEXT: ld1d { z3.d }, p1/z, [z0.d]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z2.d, p2/m, z4.d, z3.d
|
|
; CHECK-NEXT: st1d { z2.d }, p1, [z0.d]
|
|
; CHECK-NEXT: histcnt z0.d, p0/z, z1.d, z1.d
|
|
; CHECK-NEXT: ld1d { z2.d }, p0/z, [z1.d]
|
|
; CHECK-NEXT: mad z0.d, p2/m, z4.d, z2.d
|
|
; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
|
|
; CHECK-NEXT: ret
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i64_8_lane(<vscale x 8 x ptr> %buckets, i64 %inc, <vscale x 8 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i64_8_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p2.h, p0.b
|
|
; CHECK-NEXT: mov z6.d, x0
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: punpklo p3.h, p2.b
|
|
; CHECK-NEXT: punpkhi p2.h, p2.b
|
|
; CHECK-NEXT: histcnt z4.d, p3/z, z0.d, z0.d
|
|
; CHECK-NEXT: ld1d { z5.d }, p3/z, [z0.d]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z4.d, p1/m, z6.d, z5.d
|
|
; CHECK-NEXT: st1d { z4.d }, p3, [z0.d]
|
|
; CHECK-NEXT: histcnt z0.d, p2/z, z1.d, z1.d
|
|
; CHECK-NEXT: ld1d { z4.d }, p2/z, [z1.d]
|
|
; CHECK-NEXT: mad z0.d, p1/m, z6.d, z4.d
|
|
; CHECK-NEXT: st1d { z0.d }, p2, [z1.d]
|
|
; CHECK-NEXT: punpklo p2.h, p0.b
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: histcnt z0.d, p2/z, z2.d, z2.d
|
|
; CHECK-NEXT: ld1d { z1.d }, p2/z, [z2.d]
|
|
; CHECK-NEXT: mad z0.d, p1/m, z6.d, z1.d
|
|
; CHECK-NEXT: st1d { z0.d }, p2, [z2.d]
|
|
; CHECK-NEXT: histcnt z0.d, p0/z, z3.d, z3.d
|
|
; CHECK-NEXT: ld1d { z1.d }, p0/z, [z3.d]
|
|
; CHECK-NEXT: mad z0.d, p1/m, z6.d, z1.d
|
|
; CHECK-NEXT: st1d { z0.d }, p0, [z3.d]
|
|
; CHECK-NEXT: ret
|
|
call void @llvm.experimental.vector.histogram.add.nxv8p0.i64(<vscale x 8 x ptr> %buckets, i64 %inc, <vscale x 8 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i32_8_lane(ptr %base, <vscale x 8 x i32> %indices, i32 %inc, <vscale x 8 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i32_8_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.s, w1
|
|
; CHECK-NEXT: ptrue p2.s
|
|
; CHECK-NEXT: histcnt z2.s, p1/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z3.s }, p1/z, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z2.s, p2/m, z4.s, z3.s
|
|
; CHECK-NEXT: st1w { z2.s }, p1, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: histcnt z0.s, p0/z, z1.s, z1.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z1.s, sxtw #2]
|
|
; CHECK-NEXT: mad z0.s, p2/m, z4.s, z2.s
|
|
; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 8 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv8p0.i32(<vscale x 8 x ptr> %buckets, i32 %inc, <vscale x 8 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_8_lane(ptr %base, <vscale x 8 x i32> %indices, i16 %inc, <vscale x 8 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i16_8_lane:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.s, w1
|
|
; CHECK-NEXT: ptrue p2.s
|
|
; CHECK-NEXT: histcnt z2.s, p1/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1h { z3.s }, p1/z, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z2.s, p2/m, z4.s, z3.s
|
|
; CHECK-NEXT: st1h { z2.s }, p1, [x0, z0.s, sxtw #1]
|
|
; CHECK-NEXT: histcnt z0.s, p0/z, z1.s, z1.s
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z1.s, sxtw #1]
|
|
; CHECK-NEXT: mad z0.s, p2/m, z4.s, z2.s
|
|
; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 8 x i32> %indices
|
|
call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 %inc, <vscale x 8 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i8_zext(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i8 %inc) #0{
|
|
; CHECK-LABEL: histogram_i8_zext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, w1
|
|
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x0, z0.s, uxtw]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1b { z1.s }, p0, [x0, z0.s, uxtw]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i8, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i8(<vscale x 4 x ptr> %buckets, i8 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i16_zext(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i16 %inc) #0{
|
|
; CHECK-LABEL: histogram_i16_zext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, w1
|
|
; CHECK-NEXT: ld1h { z2.s }, p0/z, [x0, z0.s, uxtw #1]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1h { z1.s }, p0, [x0, z0.s, uxtw #1]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i16, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %buckets, i16 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i32_zext(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i32_zext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_i32_sext(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_i32_sext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = sext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_zext_from_i8_to_i64(ptr %base, <vscale x 4 x i8> %indices, <vscale x 4 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_zext_from_i8_to_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: and z0.s, z0.s, #0xff
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i8> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_zext_from_i16_to_i64(ptr %base, <vscale x 4 x i16> %indices, <vscale x 4 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_zext_from_i16_to_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: and z0.s, z0.s, #0xffff
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i16> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_sext_from_i16_to_i64(ptr %base, <vscale x 4 x i16> %indices, <vscale x 4 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_sext_from_i16_to_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: sxth z0.s, p1/m, z0.s
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = sext <vscale x 4 x i16> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_zext_from_i8_to_i32(ptr %base, <vscale x 4 x i8> %indices, <vscale x 4 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_zext_from_i8_to_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: and z0.s, z0.s, #0xff
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i8> %indices to <vscale x 4 x i32>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i32> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_zext_from_i16_to_i32(ptr %base, <vscale x 4 x i16> %indices, <vscale x 4 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_zext_from_i16_to_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: and z0.s, z0.s, #0xffff
|
|
; CHECK-NEXT: mov z3.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p1.s
|
|
; CHECK-NEXT: histcnt z1.s, p0/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: mad z1.s, p1/m, z3.s, z2.s
|
|
; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 4 x i16> %indices to <vscale x 4 x i32>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i32> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_2_lane_zext(ptr %base, <vscale x 2 x i32> %indices, <vscale x 2 x i1> %mask) #0 {
|
|
; CHECK-LABEL: histogram_2_lane_zext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: movprfx z1, z0
|
|
; CHECK-NEXT: and z1.d, z1.d, #0xffffffff
|
|
; CHECK-NEXT: mov z3.d, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p1.d
|
|
; CHECK-NEXT: ld1w { z2.d }, p0/z, [x0, z0.d, uxtw #2]
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z1.d, z1.d
|
|
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
|
|
; CHECK-NEXT: st1w { z1.d }, p0, [x0, z0.d, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 2 x i32> %indices to <vscale x 2 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 2 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> %buckets, i32 1, <vscale x 2 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_8_lane_zext(ptr %base, <vscale x 8 x i32> %indices, <vscale x 8 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_8_lane_zext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p2.s
|
|
; CHECK-NEXT: histcnt z2.s, p1/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z3.s }, p1/z, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z2.s, p2/m, z4.s, z3.s
|
|
; CHECK-NEXT: st1w { z2.s }, p1, [x0, z0.s, uxtw #2]
|
|
; CHECK-NEXT: histcnt z0.s, p0/z, z1.s, z1.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z1.s, uxtw #2]
|
|
; CHECK-NEXT: mad z0.s, p2/m, z4.s, z2.s
|
|
; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, uxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = zext <vscale x 8 x i32> %indices to <vscale x 8 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 8 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv8p0.i32(<vscale x 8 x ptr> %buckets, i32 1, <vscale x 8 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_8_lane_sext(ptr %base, <vscale x 8 x i32> %indices, <vscale x 8 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_8_lane_sext:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.s, #1 // =0x1
|
|
; CHECK-NEXT: ptrue p2.s
|
|
; CHECK-NEXT: histcnt z2.s, p1/z, z0.s, z0.s
|
|
; CHECK-NEXT: ld1w { z3.s }, p1/z, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: mad z2.s, p2/m, z4.s, z3.s
|
|
; CHECK-NEXT: st1w { z2.s }, p1, [x0, z0.s, sxtw #2]
|
|
; CHECK-NEXT: histcnt z0.s, p0/z, z1.s, z1.s
|
|
; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, z1.s, sxtw #2]
|
|
; CHECK-NEXT: mad z0.s, p2/m, z4.s, z2.s
|
|
; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
|
|
; CHECK-NEXT: ret
|
|
%extended = sext <vscale x 8 x i32> %indices to <vscale x 8 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 8 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv8p0.i32(<vscale x 8 x ptr> %buckets, i32 1, <vscale x 8 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_zero_mask(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_zero_mask:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: ret
|
|
call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> zeroinitializer)
|
|
ret void
|
|
}
|
|
|
|
define void @histogram_sext_zero_mask(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0{
|
|
; CHECK-LABEL: histogram_sext_zero_mask:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: ret
|
|
%extended = sext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
|
|
%buckets = getelementptr i32, ptr %base, <vscale x 4 x i64> %extended
|
|
call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> zeroinitializer)
|
|
ret void
|
|
}
|
|
|
|
; Test that we don't use nxv4i32 offsets when the increment must be performed as i64's.
|
|
define void @histogram_dont_optimize_index(ptr %p, i64 %inc, <vscale x 4 x i1> %mask, <vscale x 4 x i8> %offsets) #0 {
|
|
; CHECK-LABEL: histogram_dont_optimize_index:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: and z0.s, z0.s, #0xff
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.d, x1
|
|
; CHECK-NEXT: ptrue p2.d
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: uunpklo z1.d, z0.s
|
|
; CHECK-NEXT: uunpkhi z0.d, z0.s
|
|
; CHECK-NEXT: histcnt z2.d, p1/z, z1.d, z1.d
|
|
; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0, z1.d, lsl #3]
|
|
; CHECK-NEXT: mad z2.d, p2/m, z4.d, z3.d
|
|
; CHECK-NEXT: st1d { z2.d }, p1, [x0, z1.d, lsl #3]
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, z0.d, lsl #3]
|
|
; CHECK-NEXT: mad z1.d, p2/m, z4.d, z2.d
|
|
; CHECK-NEXT: st1d { z1.d }, p0, [x0, z0.d, lsl #3]
|
|
; CHECK-NEXT: ret
|
|
%4 = zext <vscale x 4 x i8> %offsets to <vscale x 4 x i64>
|
|
%5 = getelementptr i64, ptr %p, <vscale x 4 x i64> %4
|
|
call void @llvm.experimental.vector.histogram.add.nxv16p0.i64(<vscale x 4 x ptr> %5, i64 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
; Test that we don't use nxv4i32 offsets when the increment must be performed as i64's.
|
|
define void @histogram_sign_extend_index(ptr %p, i64 %inc, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %offsets) #0 {
|
|
; CHECK-LABEL: histogram_sign_extend_index:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: sunpklo z1.d, z0.s
|
|
; CHECK-NEXT: punpklo p1.h, p0.b
|
|
; CHECK-NEXT: mov z4.d, x1
|
|
; CHECK-NEXT: ptrue p2.d
|
|
; CHECK-NEXT: sunpkhi z0.d, z0.s
|
|
; CHECK-NEXT: punpkhi p0.h, p0.b
|
|
; CHECK-NEXT: histcnt z2.d, p1/z, z1.d, z1.d
|
|
; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0, z1.d, lsl #3]
|
|
; CHECK-NEXT: mad z2.d, p2/m, z4.d, z3.d
|
|
; CHECK-NEXT: st1d { z2.d }, p1, [x0, z1.d, lsl #3]
|
|
; CHECK-NEXT: histcnt z1.d, p0/z, z0.d, z0.d
|
|
; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, z0.d, lsl #3]
|
|
; CHECK-NEXT: mad z1.d, p2/m, z4.d, z2.d
|
|
; CHECK-NEXT: st1d { z1.d }, p0, [x0, z0.d, lsl #3]
|
|
; CHECK-NEXT: ret
|
|
%5 = getelementptr i64, ptr %p, <vscale x 4 x i32> %offsets
|
|
call void @llvm.experimental.vector.histogram.add.nxv16p0.i64(<vscale x 4 x ptr> %5, i64 %inc, <vscale x 4 x i1> %mask)
|
|
ret void
|
|
}
|
|
|
|
attributes #0 = { "target-features"="+sve2" vscale_range(1, 16) }
|