derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
546 lines
35 KiB
LLVM
546 lines
35 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: opt -mattr=+neon,+i8mm < %s -passes=msan -S | FileCheck %s
|
|
;
|
|
; Forked from llvm/test/CodeGen/AArch64/aarch64-matmul.ll
|
|
;
|
|
; Strictly handled: (none)
|
|
;
|
|
; Heuristically handled: (none)
|
|
|
|
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
|
|
target triple = "aarch64--linux-android9001"
|
|
|
|
define <4 x i32> @smmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @smmla.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
|
|
; CHECK-NEXT: [[VMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VMMLA1_I]]
|
|
;
|
|
entry:
|
|
%vmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b)
|
|
ret <4 x i32> %vmmla1.i
|
|
}
|
|
|
|
define <4 x i32> @ummla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @ummla.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
|
|
; CHECK-NEXT: [[VMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VMMLA1_I]]
|
|
;
|
|
entry:
|
|
%vmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b)
|
|
ret <4 x i32> %vmmla1.i
|
|
}
|
|
|
|
define <4 x i32> @usmmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @usmmla.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
|
|
; CHECK-NEXT: [[VUSMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSMMLA1_I]]
|
|
;
|
|
entry:
|
|
%vusmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) #3
|
|
ret <4 x i32> %vusmmla1.i
|
|
}
|
|
|
|
define <2 x i32> @usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @usdot.v2i32.v8i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP17:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <8 x i8> [[B]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP7:%.*]] = and <8 x i1> [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = and <8 x i1> [[TMP5]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP3]], [[TMP6]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = or <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <8 x i1> [[TMP10]], [[TMP9]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i1> [[TMP11]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP13:%.*]] = bitcast <8 x i8> [[TMP12]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP14:%.*]] = icmp ne <2 x i32> [[TMP13]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP15:%.*]] = sext <2 x i1> [[TMP14]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP16:%.*]] = or <2 x i32> [[TMP15]], [[TMP17]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[B]])
|
|
; CHECK-NEXT: store <2 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b)
|
|
ret <2 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <2 x i32> @usdot_lane.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @usdot_lane.v2i32.v8i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[TMP6]])
|
|
; CHECK-NEXT: store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <8 x i8> %b to <2 x i32>
|
|
%shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <2 x i32> zeroinitializer
|
|
%1 = bitcast <2 x i32> %shuffle to <8 x i8>
|
|
%vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %1)
|
|
ret <2 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <2 x i32> @sudot_lane.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @sudot_lane.v2i32.v8i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[TMP6]], <8 x i8> [[A]])
|
|
; CHECK-NEXT: store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <8 x i8> %b to <2 x i32>
|
|
%shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <2 x i32> zeroinitializer
|
|
%1 = bitcast <2 x i32> %shuffle to <8 x i8>
|
|
%vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %1, <8 x i8> %a)
|
|
ret <2 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <2 x i32> @usdot_lane.v2i32.v16i8(<2 x i32> %r, <8 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @usdot_lane.v2i32.v16i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[TMP6]])
|
|
; CHECK-NEXT: store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <16 x i8> %b to <4 x i32>
|
|
%shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> zeroinitializer
|
|
%1 = bitcast <2 x i32> %shuffle to <8 x i8>
|
|
%vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %1)
|
|
ret <2 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <2 x i32> @sudot_lane.v2i32.v16i8(<2 x i32> %r, <8 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @sudot_lane.v2i32.v16i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[TMP6]], <8 x i8> [[A]])
|
|
; CHECK-NEXT: store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <16 x i8> %b to <4 x i32>
|
|
%shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> zeroinitializer
|
|
%1 = bitcast <2 x i32> %shuffle to <8 x i8>
|
|
%vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %1, <8 x i8> %a) #3
|
|
ret <2 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <4 x i32> @usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @usdot.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP17:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <16 x i8> [[B]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP7:%.*]] = and <16 x i1> [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = and <16 x i1> [[TMP5]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = and <16 x i1> [[TMP3]], [[TMP6]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = or <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <16 x i1> [[TMP10]], [[TMP9]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i1> [[TMP11]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP13:%.*]] = bitcast <16 x i8> [[TMP12]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP14:%.*]] = icmp ne <4 x i32> [[TMP13]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP15:%.*]] = sext <4 x i1> [[TMP14]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP17]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) #3
|
|
ret <4 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <4 x i32> @usdot_lane.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @usdot_lane.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[TMP6]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <8 x i8> %b to <2 x i32>
|
|
%shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <4 x i32> zeroinitializer
|
|
%1 = bitcast <4 x i32> %shuffle to <16 x i8>
|
|
%vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %1) #3
|
|
ret <4 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <4 x i32> @sudot_lane.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @sudot_lane.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[TMP6]], <16 x i8> [[A]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <8 x i8> %b to <2 x i32>
|
|
%shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <4 x i32> zeroinitializer
|
|
%1 = bitcast <4 x i32> %shuffle to <16 x i8>
|
|
%vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %1, <16 x i8> %a) #3
|
|
ret <4 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <4 x i32> @usdot_laneq.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @usdot_laneq.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[TMP6]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <16 x i8> %b to <4 x i32>
|
|
%shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <4 x i32> zeroinitializer
|
|
%1 = bitcast <4 x i32> %shuffle to <16 x i8>
|
|
%vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %1) #3
|
|
ret <4 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <4 x i32> @sudot_laneq.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @sudot_laneq.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
|
|
; CHECK-NEXT: [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[TMP6]], <16 x i8> [[A]])
|
|
; CHECK-NEXT: store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[VUSDOT1_I]]
|
|
;
|
|
entry:
|
|
%0 = bitcast <16 x i8> %b to <4 x i32>
|
|
%shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <4 x i32> zeroinitializer
|
|
%1 = bitcast <4 x i32> %shuffle to <16 x i8>
|
|
%vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %1, <16 x i8> %a) #3
|
|
ret <4 x i32> %vusdot1.i
|
|
}
|
|
|
|
define <2 x i32> @usdot_add_zero.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <2 x i32> @usdot_add_zero.v2i32.v8i8(
|
|
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[TMP0]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <8 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <8 x i8> [[B]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP7:%.*]] = and <8 x i1> [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = and <8 x i1> [[TMP5]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP3]], [[TMP6]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = or <8 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <8 x i1> [[TMP10]], [[TMP9]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i1> [[TMP11]] to <8 x i8>
|
|
; CHECK-NEXT: [[TMP13:%.*]] = bitcast <8 x i8> [[TMP12]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP14:%.*]] = icmp ne <2 x i32> [[TMP13]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP15:%.*]] = sext <2 x i1> [[TMP14]] to <2 x i32>
|
|
; CHECK-NEXT: [[TMP16:%.*]] = or <2 x i32> [[TMP15]], zeroinitializer
|
|
; CHECK-NEXT: [[X:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> zeroinitializer, <8 x i8> [[A]], <8 x i8> [[B]])
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = or <2 x i32> [[TMP16]], [[TMP2]]
|
|
; CHECK-NEXT: [[Y:%.*]] = add <2 x i32> [[X]], [[R]]
|
|
; CHECK-NEXT: store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <2 x i32> [[Y]]
|
|
;
|
|
entry:
|
|
%x = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> zeroinitializer, <8 x i8> %a, <8 x i8> %b)
|
|
%y = add <2 x i32> %x, %r
|
|
ret <2 x i32> %y
|
|
}
|
|
|
|
define <4 x i32> @usdot_add_zero.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
|
|
; CHECK-LABEL: define <4 x i32> @usdot_add_zero.v4i32.v16i8(
|
|
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
|
|
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
|
|
; CHECK-NEXT: call void @llvm.donothing()
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <16 x i8> [[B]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP7:%.*]] = and <16 x i1> [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = and <16 x i1> [[TMP5]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = and <16 x i1> [[TMP3]], [[TMP6]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = or <16 x i1> [[TMP7]], [[TMP8]]
|
|
; CHECK-NEXT: [[TMP11:%.*]] = or <16 x i1> [[TMP10]], [[TMP9]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i1> [[TMP11]] to <16 x i8>
|
|
; CHECK-NEXT: [[TMP13:%.*]] = bitcast <16 x i8> [[TMP12]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP14:%.*]] = icmp ne <4 x i32> [[TMP13]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP15:%.*]] = sext <4 x i1> [[TMP14]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP16:%.*]] = or <4 x i32> [[TMP15]], zeroinitializer
|
|
; CHECK-NEXT: [[X:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[A]], <16 x i8> [[B]])
|
|
; CHECK-NEXT: [[_MSPROP:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
|
|
; CHECK-NEXT: [[Y:%.*]] = add <4 x i32> [[X]], [[R]]
|
|
; CHECK-NEXT: store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
|
|
; CHECK-NEXT: ret <4 x i32> [[Y]]
|
|
;
|
|
entry:
|
|
%x = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> %a, <16 x i8> %b)
|
|
%y = add <4 x i32> %x, %r
|
|
ret <4 x i32> %y
|
|
}
|
|
|
|
declare <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
|
|
declare <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
|
|
declare <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
|
|
declare <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>) #2
|
|
declare <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
|