; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve-bfscale -force-streaming -verify-machineinstrs < %s | FileCheck %s

define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_scale_single_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x i16> %zm) {
; CHECK-LABEL: multi_vec_scale_single_x2_bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    bfscale { z0.h, z1.h }, { z0.h, z1.h }, z2.h
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fscale.single.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x i16> %zm)
  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
}

define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_scale_single_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x i16> %zm) {
; CHECK-LABEL: multi_vec_scale_single_x4_bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    bfscale { z0.h - z3.h }, { z0.h - z3.h }, z4.h
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fscale.single.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x i16> %zm)
  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
}

define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_scale_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2) {
; CHECK-LABEL: multi_vec_scale_x2_bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    bfscale { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fscale.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2)
  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
}

define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_scale_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3, <vscale x 8 x i16> %zm4) {
; CHECK-LABEL: multi_vec_scale_x4_bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    bfscale { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
; CHECK-NEXT:    ret
  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fscale.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3, <vscale x 8 x i16> %zm4)
  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
}

define <vscale x 8 x bfloat> @bfscale_h(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: bfscale_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    bfscale z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fscale.nxv8bf16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x bfloat> %a,
                                                                      <vscale x 8 x i16> %b)
  ret <vscale x 8 x bfloat> %out
}