; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mattr=+neon,+i8mm < %s -passes=msan -S | FileCheck %s
;
; Forked from llvm/test/CodeGen/AArch64/aarch64-matmul.ll
;
; Strictly handled: (none)
;
; Heuristically handled: (none)

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-android9001"

define <4 x i32> @smmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @smmla.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
; CHECK-NEXT:    [[VMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VMMLA1_I]]
;
entry:
  %vmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b)
  ret <4 x i32> %vmmla1.i
}

define <4 x i32> @ummla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @ummla.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
; CHECK-NEXT:    [[VMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VMMLA1_I]]
;
entry:
  %vmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b)
  ret <4 x i32> %vmmla1.i
}

define <4 x i32> @usmmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @usmmla.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <16 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = zext <16 x i1> [[TMP5]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[TMP4]], <16 x i8> [[TMP6]])
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <4 x i32> [[TMP7]], splat (i32 8)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <4 x i32> [[TMP0]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = or <4 x i1> [[TMP8]], [[TMP10]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <4 x i1> [[TMP11]] to <4 x i32>
; CHECK-NEXT:    [[VUSMMLA1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSMMLA1_I]]
;
entry:
  %vusmmla1.i = tail call <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) #3
  ret <4 x i32> %vusmmla1.i
}

define <2 x i32> @usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @usdot.v2i32.v8i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP17:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <8 x i8> [[B]], zeroinitializer
; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i1> [[TMP5]], [[TMP4]]
; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i1> [[TMP3]], [[TMP6]]
; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i1> [[TMP10]], [[TMP9]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <8 x i1> [[TMP11]] to <8 x i8>
; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i8> [[TMP12]] to <2 x i32>
; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <2 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT:    [[TMP15:%.*]] = sext <2 x i1> [[TMP14]] to <2 x i32>
; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i32> [[TMP15]], [[TMP17]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[B]])
; CHECK-NEXT:    store <2 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[VUSDOT1_I]]
;
entry:
  %vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b)
  ret <2 x i32> %vusdot1.i
}

define <2 x i32> @usdot_lane.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @usdot_lane.v2i32.v8i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <2 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[TMP6]])
; CHECK-NEXT:    store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <8 x i8> %b to <2 x i32>
  %shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <2 x i32> zeroinitializer
  %1 = bitcast <2 x i32> %shuffle to <8 x i8>
  %vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %1)
  ret <2 x i32> %vusdot1.i
}

define <2 x i32> @sudot_lane.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @sudot_lane.v2i32.v8i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <2 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[TMP6]], <8 x i8> [[A]])
; CHECK-NEXT:    store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <8 x i8> %b to <2 x i32>
  %shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <2 x i32> zeroinitializer
  %1 = bitcast <2 x i32> %shuffle to <8 x i8>
  %vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %1, <8 x i8> %a)
  ret <2 x i32> %vusdot1.i
}

define <2 x i32> @usdot_lane.v2i32.v16i8(<2 x i32> %r, <8 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @usdot_lane.v2i32.v16i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <2 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[A]], <8 x i8> [[TMP6]])
; CHECK-NEXT:    store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <16 x i8> %b to <4 x i32>
  %shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> zeroinitializer
  %1 = bitcast <2 x i32> %shuffle to <8 x i8>
  %vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %1)
  ret <2 x i32> %vusdot1.i
}

define <2 x i32> @sudot_lane.v2i32.v16i8(<2 x i32> %r, <8 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @sudot_lane.v2i32.v16i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <2 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[_MSPROP]] to <8 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[SHUFFLE]] to <8 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP15]] to <8 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP16]] to <2 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <2 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <2 x i1> [[TMP18]] to <2 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> [[R]], <8 x i8> [[TMP6]], <8 x i8> [[A]])
; CHECK-NEXT:    store <2 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <16 x i8> %b to <4 x i32>
  %shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> zeroinitializer
  %1 = bitcast <2 x i32> %shuffle to <8 x i8>
  %vusdot1.i = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> %r, <8 x i8> %1, <8 x i8> %a) #3
  ret <2 x i32> %vusdot1.i
}

define <4 x i32> @usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @usdot.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP17:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i8> [[B]], zeroinitializer
; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i1> [[TMP5]], [[TMP4]]
; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i1> [[TMP3]], [[TMP6]]
; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i1> [[TMP10]], [[TMP9]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <16 x i1> [[TMP11]] to <16 x i8>
; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i8> [[TMP12]] to <4 x i32>
; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <4 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i1> [[TMP14]] to <4 x i32>
; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP17]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[B]])
; CHECK-NEXT:    store <4 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSDOT1_I]]
;
entry:
  %vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) #3
  ret <4 x i32> %vusdot1.i
}

define <4 x i32> @usdot_lane.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @usdot_lane.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <4 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[TMP6]])
; CHECK-NEXT:    store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <8 x i8> %b to <2 x i32>
  %shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <4 x i32> zeroinitializer
  %1 = bitcast <4 x i32> %shuffle to <16 x i8>
  %vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %1) #3
  ret <4 x i32> %vusdot1.i
}

define <4 x i32> @sudot_lane.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @sudot_lane.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[B]] to <2 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> splat (i32 -1), <4 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[TMP6]], <16 x i8> [[A]])
; CHECK-NEXT:    store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <8 x i8> %b to <2 x i32>
  %shuffle = shufflevector <2 x i32> %0, <2 x i32> poison, <4 x i32> zeroinitializer
  %1 = bitcast <4 x i32> %shuffle to <16 x i8>
  %vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %1, <16 x i8> %a) #3
  ret <4 x i32> %vusdot1.i
}

define <4 x i32> @usdot_laneq.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @usdot_laneq.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <4 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[A]], <16 x i8> [[TMP6]])
; CHECK-NEXT:    store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <16 x i8> %b to <4 x i32>
  %shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <4 x i32> zeroinitializer
  %1 = bitcast <4 x i32> %shuffle to <16 x i8>
  %vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %1) #3
  ret <4 x i32> %vusdot1.i
}

define <4 x i32> @sudot_laneq.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @sudot_laneq.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[B]] to <4 x i32>
; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> splat (i32 -1), <4 x i32> zeroinitializer
; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[_MSPROP]] to <16 x i8>
; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[SHUFFLE]] to <16 x i8>
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP2]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP9]], [[TMP8]]
; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP7]], [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i1> [[TMP11]], [[TMP12]]
; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP14]], [[TMP13]]
; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP15]] to <16 x i8>
; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <16 x i8> [[TMP16]] to <4 x i32>
; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT:    [[TMP19:%.*]] = sext <4 x i1> [[TMP18]] to <4 x i32>
; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP19]], [[TMP21]]
; CHECK-NEXT:    [[VUSDOT1_I:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[R]], <16 x i8> [[TMP6]], <16 x i8> [[A]])
; CHECK-NEXT:    store <4 x i32> [[TMP20]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[VUSDOT1_I]]
;
entry:
  %0 = bitcast <16 x i8> %b to <4 x i32>
  %shuffle = shufflevector <4 x i32> %0, <4 x i32> poison, <4 x i32> zeroinitializer
  %1 = bitcast <4 x i32> %shuffle to <16 x i8>
  %vusdot1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %r, <16 x i8> %1, <16 x i8> %a) #3
  ret <4 x i32> %vusdot1.i
}

define <2 x i32> @usdot_add_zero.v2i32.v8i8(<2 x i32> %r, <8 x i8> %a, <8 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <2 x i32> @usdot_add_zero.v2i32.v8i8(
; CHECK-SAME: <2 x i32> [[R:%.*]], <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 8), align 8
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i8> [[TMP0]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <8 x i8> [[B]], zeroinitializer
; CHECK-NEXT:    [[TMP7:%.*]] = and <8 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[TMP8:%.*]] = and <8 x i1> [[TMP5]], [[TMP4]]
; CHECK-NEXT:    [[TMP9:%.*]] = and <8 x i1> [[TMP3]], [[TMP6]]
; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP11:%.*]] = or <8 x i1> [[TMP10]], [[TMP9]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <8 x i1> [[TMP11]] to <8 x i8>
; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i8> [[TMP12]] to <2 x i32>
; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <2 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT:    [[TMP15:%.*]] = sext <2 x i1> [[TMP14]] to <2 x i32>
; CHECK-NEXT:    [[TMP16:%.*]] = or <2 x i32> [[TMP15]], zeroinitializer
; CHECK-NEXT:    [[X:%.*]] = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> zeroinitializer, <8 x i8> [[A]], <8 x i8> [[B]])
; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP16]], [[TMP2]]
; CHECK-NEXT:    [[Y:%.*]] = add <2 x i32> [[X]], [[R]]
; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <2 x i32> [[Y]]
;
entry:
  %x = tail call <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32> zeroinitializer, <8 x i8> %a, <8 x i8> %b)
  %y = add <2 x i32> %x, %r
  ret <2 x i32> %y
}

define <4 x i32> @usdot_add_zero.v4i32.v16i8(<4 x i32> %r, <16 x i8> %a, <16 x i8> %b) sanitize_memory {
; CHECK-LABEL: define <4 x i32> @usdot_add_zero.v4i32.v16i8(
; CHECK-SAME: <4 x i32> [[R:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 16), align 8
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr getelementptr (i8, ptr @__msan_param_tls, i64 32), align 8
; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i8> [[A]], zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i8> [[B]], zeroinitializer
; CHECK-NEXT:    [[TMP7:%.*]] = and <16 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[TMP8:%.*]] = and <16 x i1> [[TMP5]], [[TMP4]]
; CHECK-NEXT:    [[TMP9:%.*]] = and <16 x i1> [[TMP3]], [[TMP6]]
; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i1> [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i1> [[TMP10]], [[TMP9]]
; CHECK-NEXT:    [[TMP12:%.*]] = sext <16 x i1> [[TMP11]] to <16 x i8>
; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x i8> [[TMP12]] to <4 x i32>
; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <4 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i1> [[TMP14]] to <4 x i32>
; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], zeroinitializer
; CHECK-NEXT:    [[X:%.*]] = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> [[A]], <16 x i8> [[B]])
; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
; CHECK-NEXT:    [[Y:%.*]] = add <4 x i32> [[X]], [[R]]
; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <4 x i32> [[Y]]
;
entry:
  %x = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> %a, <16 x i8> %b)
  %y = add <4 x i32> %x, %r
  ret <4 x i32> %y
}

declare <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
declare <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
declare <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2
declare <2 x i32> @llvm.aarch64.neon.usdot.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>) #2
declare <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32>, <16 x i8>, <16 x i8>) #2