derek/TheRock
1
0 Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
1a182e2dac58879f215c0fad0f5f78c76dcafe56
TheRock/compiler/amd-llvm/llvm/test/CodeGen/X86/avx10.2-fma-commute.ll
T
derek.holloway 7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
Details
CodeQL / Analyze (python) (push) Canceled after 0s
Details
pre-commit / pre-commit (push) Canceled after 0s
Details
Multi-Arch CI / setup (push) Canceled after 0s
Details
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Details
Setup (Multi-Arch) / setup (push) Canceled after 0s
Details
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Details
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Details
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Details
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Details
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Details
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
Details
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Details
Multi-Arch CI / CI Summary (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Details
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Details
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Details
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Details
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Details
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Details
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Details
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
Details
Flatten subprojects into a single base monorepo
2026-09-02 16:34:47 -07:00

1245 lines
50 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s --mtriple=x86_64-unknown-unknown -mattr=avx10.2 | FileCheck %s
define <8 x bfloat> @fma_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {
; CHECK-LABEL: fma_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
ret <8 x bfloat> %a
}
define <8 x bfloat> @fma_mask_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %xmm1, %xmm2, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %xmm2, %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_mask_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {
; CHECK-LABEL: fma_maskz_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_mask_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_mask_load_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_123_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_213_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_231_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_321_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_132_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <8 x bfloat> @fma_maskz_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {
; CHECK-LABEL: fma_maskz_load_312_v8bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <8 x bfloat>, ptr %zp
%a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)
%b = bitcast i8 %mask to <8 x i1>
%c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer
ret <8 x bfloat> %c
}
define <16 x bfloat> @fma_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {
; CHECK-LABEL: fma_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
ret <16 x bfloat> %a
}
define <16 x bfloat> @fma_mask_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %ymm1, %ymm2, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %ymm2, %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %ymm2, %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_mask_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {
; CHECK-LABEL: fma_maskz_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_mask_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_mask_load_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_123_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_213_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_231_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_321_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_132_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <16 x bfloat> @fma_maskz_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {
; CHECK-LABEL: fma_maskz_load_312_v16bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <16 x bfloat>, ptr %zp
%a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)
%b = bitcast i16 %mask to <16 x i1>
%c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer
ret <16 x bfloat> %c
}
define <32 x bfloat> @fma_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {
; CHECK-LABEL: fma_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {
; CHECK-LABEL: fma_load_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
ret <32 x bfloat> %a
}
define <32 x bfloat> @fma_mask_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %zmm1, %zmm2, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %zmm2, %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd132bf16 %zmm2, %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_mask_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {
; CHECK-LABEL: fma_maskz_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %edi, %k1
; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_mask_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_mask_load_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_123_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_213_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_231_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_321_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_132_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
define <32 x bfloat> @fma_maskz_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {
; CHECK-LABEL: fma_maskz_load_312_v32bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}
; CHECK-NEXT: retq
%z = load <32 x bfloat>, ptr %zp
%a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)
%b = bitcast i32 %mask to <32 x i1>
%c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer
ret <32 x bfloat> %c
}
Reference in new issue View Git Blame Copy Permalink