; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mattr=+sve,+i8mm < %s -passes=msan -S | FileCheck %s
;
; Forked from llvm/test/CodeGen/AArch64/sve-intrinsics-matmul-int8.ll
;
; Strictly handled:
; - llvm.aarch64.sve.smmla.nxv4i32
; - llvm.aarch64.sve.ummla.nxv4i32
; - llvm.aarch64.sve.usmmla.nxv4i32
; - llvm.aarch64.sve.usdot.nxv4i32
; - llvm.aarch64.sve.usdot.lane.nxv4i32
; - llvm.aarch64.sve.sudot.lane.nxv4i32
;
; Heuristically handled: (none)

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-android9001"

define <vscale x 4 x i32> @smmla(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @smmla(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]])
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @ummla(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @ummla(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ummla.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]])
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ummla.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usmmla(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usmmla(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usmmla.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]])
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usmmla.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usdot(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usdot(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]])
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usdot_lane_0(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usdot_lane_0(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 0)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 0)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usdot_lane_1(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usdot_lane_1(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 1)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 1)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usdot_lane_2(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usdot_lane_2(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 2)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 2)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @usdot_lane_3(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @usdot_lane_3(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 3)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 3)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @sudot_lane_0(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @sudot_lane_0(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 0)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 0)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @sudot_lane_1(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @sudot_lane_1(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 1)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 1)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @sudot_lane_2(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @sudot_lane_2(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 2)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 2)
  ret <vscale x 4 x i32> %val
}

define <vscale x 4 x i32> @sudot_lane_3(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) nounwind sanitize_memory {
; CHECK-LABEL: define <vscale x 4 x i32> @sudot_lane_3(
; CHECK-SAME: <vscale x 4 x i32> [[R:%.*]], <vscale x 16 x i8> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    call void @llvm.donothing()
; CHECK-NEXT:    [[VAL:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> [[R]], <vscale x 16 x i8> [[A]], <vscale x 16 x i8> [[B]], i32 3)
; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
; CHECK-NEXT:    ret <vscale x 4 x i32> [[VAL]]
;
entry:
  %val = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %r, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 3)
  ret <vscale x 4 x i32> %val
}


declare <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.ummla.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.usmmla.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)

declare <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)
declare <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)