; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mattr=+sve,+bf16                         < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-NONSTREAMING
; RUN: llc -mattr=+sve,+bf16,+sve-b16b16             < %s | FileCheck %s --check-prefixes=CHECK,B16B16
; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-STREAMING
; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16

target triple = "aarch64-unknown-linux-gnu"

;
; FABS
;

define <vscale x 2 x bfloat> @fabs_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: fabs_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fabs_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: fabs_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fabs_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fabs_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FADD
;

define <vscale x 2 x bfloat> @fadd_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fadd_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fadd_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fadd <vscale x 2 x bfloat> %a, %b
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fadd_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fadd z0.s, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fadd_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fadd <vscale x 4 x bfloat> %a, %b
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fadd_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fadd z2.s, z4.s, z3.s
; NOB16B16-NEXT:    fadd z0.s, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fadd_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    bfadd z0.h, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fadd <vscale x 8 x bfloat> %a, %b
  ret <vscale x 8 x bfloat> %res
}

;
; FDIV
;

define <vscale x 2 x bfloat> @fdiv_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; CHECK-LABEL: fdiv_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = fdiv <vscale x 2 x bfloat> %a, %b
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fdiv_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; CHECK-LABEL: fdiv_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = fdiv <vscale x 4 x bfloat> %a, %b
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fdiv_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fdiv_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    movi v2.2d, #0000000000000000
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    zip2 z3.h, z2.h, z1.h
; CHECK-NEXT:    zip2 z4.h, z2.h, z0.h
; CHECK-NEXT:    zip1 z1.h, z2.h, z1.h
; CHECK-NEXT:    zip1 z0.h, z2.h, z0.h
; CHECK-NEXT:    fdivr z3.s, p0/m, z3.s, z4.s
; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z3.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = fdiv <vscale x 8 x bfloat> %a, %b
  ret <vscale x 8 x bfloat> %res
}

;
; FMAX
;

define <vscale x 2 x bfloat> @fmax_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fmax_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmax_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfmax z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.maximum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fmax_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fmax_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmax_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfmax z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.maximum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmax_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmax_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fmax z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmax_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.h
; B16B16-NEXT:    bfmax z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
  ret <vscale x 8 x bfloat> %res
}

;
; FMAXNM
;

define <vscale x 2 x bfloat> @fmaxnm_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fmaxnm_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmaxnm_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfmaxnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.maxnum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fmaxnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fmaxnm_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmaxnm_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfmaxnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.maxnum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmaxnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmaxnm_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fmaxnm z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmaxnm_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.h
; B16B16-NEXT:    bfmaxnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
  ret <vscale x 8 x bfloat> %res
}

;
; FMIN
;

define <vscale x 2 x bfloat> @fmin_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fmin_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmin_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfmin z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.minimum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fmin_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fmin_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmin_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfmin z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.minimum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmin_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmin_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fmin z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmin_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.h
; B16B16-NEXT:    bfmin z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
  ret <vscale x 8 x bfloat> %res
}

;
; FMINNM
;

define <vscale x 2 x bfloat> @fminnm_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fminnm_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fminnm_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfminnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.minnum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fminnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fminnm_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fminnm_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfminnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.minnum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fminnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fminnm_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fminnm z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fminnm_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.h
; B16B16-NEXT:    bfminnm z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
  ret <vscale x 8 x bfloat> %res
}

;
; FMLA
;

define <vscale x 2 x bfloat> @fmla_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c) {
; NOB16B16-LABEL: fmla_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    lsl z2.s, z2.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmla_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; B16B16-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.fma.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c) {
; NOB16B16-LABEL: fmla_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z2.s, z2.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmla_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; B16B16-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.fma.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) {
; NOB16B16-LABEL: fmla_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v3.2d, #0000000000000000
; NOB16B16-NEXT:    uunpkhi z4.s, z1.h
; NOB16B16-NEXT:    uunpkhi z5.s, z0.h
; NOB16B16-NEXT:    uunpklo z1.s, z1.h
; NOB16B16-NEXT:    uunpklo z0.s, z0.h
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z6.h, z3.h, z2.h
; NOB16B16-NEXT:    zip1 z2.h, z3.h, z2.h
; NOB16B16-NEXT:    bfmlalb z6.s, z5.h, z4.h
; NOB16B16-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z6.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT:    uzp1 z0.h, z1.h, z0.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmla_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.h
; B16B16-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; B16B16-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)
  ret <vscale x 8 x bfloat> %res
}

;
; FMUL
;

define <vscale x 2 x bfloat> @fmul_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fmul_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmul_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfmul z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fmul nsz <vscale x 2 x bfloat> %a, %b
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fmul_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-NONSTREAMING-LABEL: fmul_nxv4bf16:
; NOB16B16-NONSTREAMING:       // %bb.0:
; NOB16B16-NONSTREAMING-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NONSTREAMING-NEXT:    ptrue p0.s
; NOB16B16-NONSTREAMING-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-NONSTREAMING-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-NONSTREAMING-NEXT:    ret
;
; B16B16-LABEL: fmul_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfmul z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
;
; NOB16B16-STREAMING-LABEL: fmul_nxv4bf16:
; NOB16B16-STREAMING:       // %bb.0:
; NOB16B16-STREAMING-NEXT:    mov z2.s, #0 // =0x0
; NOB16B16-STREAMING-NEXT:    ptrue p0.s
; NOB16B16-STREAMING-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-STREAMING-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-STREAMING-NEXT:    ret
  %res = fmul nsz <vscale x 4 x bfloat> %a, %b
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-NONSTREAMING-LABEL: fmul_nxv8bf16:
; NOB16B16-NONSTREAMING:       // %bb.0:
; NOB16B16-NONSTREAMING-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NONSTREAMING-NEXT:    movi v3.2d, #0000000000000000
; NOB16B16-NONSTREAMING-NEXT:    ptrue p0.s
; NOB16B16-NONSTREAMING-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-NONSTREAMING-NEXT:    bfmlalt z3.s, z0.h, z1.h
; NOB16B16-NONSTREAMING-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-NONSTREAMING-NEXT:    bfcvtnt z0.h, p0/m, z3.s
; NOB16B16-NONSTREAMING-NEXT:    ret
;
; B16B16-LABEL: fmul_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    bfmul z0.h, z0.h, z1.h
; B16B16-NEXT:    ret
;
; NOB16B16-STREAMING-LABEL: fmul_nxv8bf16:
; NOB16B16-STREAMING:       // %bb.0:
; NOB16B16-STREAMING-NEXT:    mov z2.s, #0 // =0x0
; NOB16B16-STREAMING-NEXT:    mov z3.s, #0 // =0x0
; NOB16B16-STREAMING-NEXT:    ptrue p0.s
; NOB16B16-STREAMING-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-STREAMING-NEXT:    bfmlalt z3.s, z0.h, z1.h
; NOB16B16-STREAMING-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-STREAMING-NEXT:    bfcvtnt z0.h, p0/m, z3.s
; NOB16B16-STREAMING-NEXT:    ret
  %res = fmul nsz <vscale x 8 x bfloat> %a, %b
  ret <vscale x 8 x bfloat> %res
}

define <vscale x 8 x bfloat> @fmul_nxv8bf16_no_nsz(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmul_nxv8bf16_no_nsz:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    mov z2.s, #0x80000000
; NOB16B16-NEXT:    mov z3.s, #0x80000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    bfmlalb z2.s, z0.h, z1.h
; NOB16B16-NEXT:    bfmlalt z3.s, z0.h, z1.h
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z2.s
; NOB16B16-NEXT:    bfcvtnt z0.h, p0/m, z3.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fmul_nxv8bf16_no_nsz:
; B16B16:       // %bb.0:
; B16B16-NEXT:    bfmul z0.h, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fmul <vscale x 8 x bfloat> %a, %b
  ret <vscale x 8 x bfloat> %res
}

;
; FNEG
;

define <vscale x 2 x bfloat> @fneg_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: fneg_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = fneg <vscale x 2 x bfloat> %a
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fneg_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: fneg_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = fneg <vscale x 4 x bfloat> %a
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fneg_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fneg_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
; CHECK-NEXT:    ret
  %res = fneg <vscale x 8 x bfloat> %a
  ret <vscale x 8 x bfloat> %res
}

;
; FSQRT
;

define <vscale x 2 x bfloat> @fsqrt_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: fsqrt_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.sqrt.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fsqrt_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: fsqrt_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.sqrt.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fsqrt_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fsqrt_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    movi v1.2d, #0000000000000000
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    zip2 z2.h, z1.h, z0.h
; CHECK-NEXT:    zip1 z0.h, z1.h, z0.h
; CHECK-NEXT:    fsqrt z2.s, p0/m, z2.s
; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FSUB
;

define <vscale x 2 x bfloat> @fsub_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
; NOB16B16-LABEL: fsub_nxv2bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.d
; NOB16B16-NEXT:    fsub z0.s, p0/m, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fsub_nxv2bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.d
; B16B16-NEXT:    bfsub z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fsub <vscale x 2 x bfloat> %a, %b
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fsub_nxv4bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    lsl z1.s, z1.s, #16
; NOB16B16-NEXT:    lsl z0.s, z0.s, #16
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    fsub z0.s, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fsub_nxv4bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    ptrue p0.s
; B16B16-NEXT:    bfsub z0.h, p0/m, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fsub <vscale x 4 x bfloat> %a, %b
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @fsub_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fsub_nxv8bf16:
; NOB16B16:       // %bb.0:
; NOB16B16-NEXT:    movi v2.2d, #0000000000000000
; NOB16B16-NEXT:    ptrue p0.s
; NOB16B16-NEXT:    zip2 z3.h, z2.h, z1.h
; NOB16B16-NEXT:    zip2 z4.h, z2.h, z0.h
; NOB16B16-NEXT:    zip1 z1.h, z2.h, z1.h
; NOB16B16-NEXT:    zip1 z0.h, z2.h, z0.h
; NOB16B16-NEXT:    fsub z2.s, z4.s, z3.s
; NOB16B16-NEXT:    fsub z0.s, z0.s, z1.s
; NOB16B16-NEXT:    bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT:    bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT:    uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT:    ret
;
; B16B16-LABEL: fsub_nxv8bf16:
; B16B16:       // %bb.0:
; B16B16-NEXT:    bfsub z0.h, z0.h, z1.h
; B16B16-NEXT:    ret
  %res = fsub <vscale x 8 x bfloat> %a, %b
  ret <vscale x 8 x bfloat> %res
}

declare <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.fma.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.fma.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.maximum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.maximum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.maxnum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.maxnum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.minimum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.minimum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.minnum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.minnum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.sqrt.nxv2bf16(<vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.sqrt.nxv4bf16(<vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat>)