derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
230 lines
10 KiB
LLVM
230 lines
10 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512,AVX512DQ
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512DQBW
|
|
|
|
declare <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x float>)
|
|
declare <16 x float> @llvm.masked.expandload.v16f32(ptr, <16 x i1>, <16 x float>)
|
|
declare <8 x float> @llvm.masked.expandload.v8f32(ptr, <8 x i1>, <8 x float>)
|
|
declare <16 x i32> @llvm.masked.expandload.v16i32(ptr, <16 x i1>, <16 x i32>)
|
|
|
|
; Test case 1: Direct v8i1 all-ones mask (should use kxnorb on AVX512DQ)
|
|
define <8 x float> @mask_v8i1_allones(ptr %ptr) {
|
|
; AVX512F-LABEL: mask_v8i1_allones:
|
|
; AVX512F: # %bb.0:
|
|
; AVX512F-NEXT: movw $255, %ax
|
|
; AVX512F-NEXT: kmovw %eax, %k1
|
|
; AVX512F-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
|
|
; AVX512F-NEXT: retq
|
|
;
|
|
; AVX512DQ-LABEL: mask_v8i1_allones:
|
|
; AVX512DQ: # %bb.0:
|
|
; AVX512DQ-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQ-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
|
|
; AVX512DQ-NEXT: retq
|
|
;
|
|
; AVX512BW-LABEL: mask_v8i1_allones:
|
|
; AVX512BW: # %bb.0:
|
|
; AVX512BW-NEXT: movw $255, %ax
|
|
; AVX512BW-NEXT: kmovd %eax, %k1
|
|
; AVX512BW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
|
|
; AVX512BW-NEXT: retq
|
|
;
|
|
; AVX512DQBW-LABEL: mask_v8i1_allones:
|
|
; AVX512DQBW: # %bb.0:
|
|
; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQBW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512DQBW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
|
|
; AVX512DQBW-NEXT: retq
|
|
%res = call <8 x float> @llvm.masked.expandload.v8f32(ptr %ptr, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> zeroinitializer)
|
|
ret <8 x float> %res
|
|
}
|
|
|
|
; Test case 2: v16i1 with lower 8 bits set via bitconvert (should use kxnorb on AVX512DQ)
|
|
define <16 x float> @mask_v16i1_lower8(ptr %ptr) {
|
|
; AVX512F-LABEL: mask_v16i1_lower8:
|
|
; AVX512F: # %bb.0:
|
|
; AVX512F-NEXT: movw $255, %ax
|
|
; AVX512F-NEXT: kmovw %eax, %k1
|
|
; AVX512F-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512F-NEXT: retq
|
|
;
|
|
; AVX512DQ-LABEL: mask_v16i1_lower8:
|
|
; AVX512DQ: # %bb.0:
|
|
; AVX512DQ-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQ-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512DQ-NEXT: retq
|
|
;
|
|
; AVX512BW-LABEL: mask_v16i1_lower8:
|
|
; AVX512BW: # %bb.0:
|
|
; AVX512BW-NEXT: movw $255, %ax
|
|
; AVX512BW-NEXT: kmovd %eax, %k1
|
|
; AVX512BW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512BW-NEXT: retq
|
|
;
|
|
; AVX512DQBW-LABEL: mask_v16i1_lower8:
|
|
; AVX512DQBW: # %bb.0:
|
|
; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQBW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}
|
|
; AVX512DQBW-NEXT: retq
|
|
%res = call <16 x float> @llvm.masked.expandload.v16f32(ptr %ptr, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x float> zeroinitializer)
|
|
ret <16 x float> %res
|
|
}
|
|
|
|
; Test case 3: v16i1 with all bits set (should use kxnorw on all targets)
|
|
define <16 x float> @gather_all(ptr %base, <16 x i32> %ind, i16 %mask) {
|
|
; AVX512-LABEL: gather_all:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: kxnorw %k0, %k0, %k1
|
|
; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
|
|
; AVX512-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
|
|
; AVX512-NEXT: vmovaps %zmm1, %zmm0
|
|
; AVX512-NEXT: retq
|
|
%broadcast.splatinsert = insertelement <16 x ptr> poison, ptr %base, i32 0
|
|
%broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
|
|
%sext_ind = sext <16 x i32> %ind to <16 x i64>
|
|
%gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
|
|
%res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> poison)
|
|
ret <16 x float> %res
|
|
}
|
|
|
|
; Test case 4: v8i1 with lower 8 bits set in gather (should use kxnorb on AVX512DQ targets)
|
|
define <16 x float> @gather_lower(ptr %base, <16 x i32> %ind, i16 %mask) {
|
|
; AVX512F-LABEL: gather_lower:
|
|
; AVX512F: # %bb.0:
|
|
; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
|
|
; AVX512F-NEXT: movw $255, %ax
|
|
; AVX512F-NEXT: kmovw %eax, %k1
|
|
; AVX512F-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
|
|
; AVX512F-NEXT: vmovaps %zmm1, %zmm0
|
|
; AVX512F-NEXT: retq
|
|
;
|
|
; AVX512DQ-LABEL: gather_lower:
|
|
; AVX512DQ: # %bb.0:
|
|
; AVX512DQ-NEXT: vxorps %xmm1, %xmm1, %xmm1
|
|
; AVX512DQ-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQ-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
|
|
; AVX512DQ-NEXT: vmovaps %zmm1, %zmm0
|
|
; AVX512DQ-NEXT: retq
|
|
;
|
|
; AVX512BW-LABEL: gather_lower:
|
|
; AVX512BW: # %bb.0:
|
|
; AVX512BW-NEXT: vxorps %xmm1, %xmm1, %xmm1
|
|
; AVX512BW-NEXT: movw $255, %ax
|
|
; AVX512BW-NEXT: kmovd %eax, %k1
|
|
; AVX512BW-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
|
|
; AVX512BW-NEXT: vmovaps %zmm1, %zmm0
|
|
; AVX512BW-NEXT: retq
|
|
;
|
|
; AVX512DQBW-LABEL: gather_lower:
|
|
; AVX512DQBW: # %bb.0:
|
|
; AVX512DQBW-NEXT: vxorps %xmm1, %xmm1, %xmm1
|
|
; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k1
|
|
; AVX512DQBW-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
|
|
; AVX512DQBW-NEXT: vmovaps %zmm1, %zmm0
|
|
; AVX512DQBW-NEXT: retq
|
|
%broadcast.splatinsert = insertelement <16 x ptr> poison, ptr %base, i32 0
|
|
%broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
|
|
%sext_ind = sext <16 x i32> %ind to <16 x i64>
|
|
%gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
|
|
%res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x float> poison)
|
|
ret <16 x float> %res
|
|
}
|
|
|
|
; Test case 5: v32i1 mask via bitconvert combined with dynamic condition.
|
|
; Ensures lower 16 lanes force the KSET1W path without folding into a shuffle.
|
|
define <32 x i16> @mask_v32i1_lower16(<32 x i16> %a, <32 x i16> %b, <32 x i16> %c, <32 x i16> %d) {
|
|
; AVX512F-LABEL: mask_v32i1_lower16:
|
|
; AVX512F: # %bb.0:
|
|
; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3
|
|
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm2
|
|
; AVX512F-NEXT: vpcmpgtw %ymm3, %ymm2, %ymm2
|
|
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = -1
|
|
; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
|
|
; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))
|
|
; AVX512F-NEXT: retq
|
|
;
|
|
; AVX512DQ-LABEL: mask_v32i1_lower16:
|
|
; AVX512DQ: # %bb.0:
|
|
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm3
|
|
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm2
|
|
; AVX512DQ-NEXT: vpcmpgtw %ymm3, %ymm2, %ymm2
|
|
; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm3 = -1
|
|
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
|
|
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))
|
|
; AVX512DQ-NEXT: retq
|
|
;
|
|
; AVX512BW-LABEL: mask_v32i1_lower16:
|
|
; AVX512BW: # %bb.0:
|
|
; AVX512BW-NEXT: movl $65535, %eax # imm = 0xFFFF
|
|
; AVX512BW-NEXT: kmovd %eax, %k0
|
|
; AVX512BW-NEXT: vpcmpgtw %zmm3, %zmm2, %k1
|
|
; AVX512BW-NEXT: kord %k0, %k1, %k1
|
|
; AVX512BW-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
|
|
; AVX512BW-NEXT: retq
|
|
;
|
|
; AVX512DQBW-LABEL: mask_v32i1_lower16:
|
|
; AVX512DQBW: # %bb.0:
|
|
; AVX512DQBW-NEXT: kxnorw %k0, %k0, %k0
|
|
; AVX512DQBW-NEXT: vpcmpgtw %zmm3, %zmm2, %k1
|
|
; AVX512DQBW-NEXT: kord %k0, %k1, %k1
|
|
; AVX512DQBW-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
|
|
; AVX512DQBW-NEXT: retq
|
|
%mask0 = bitcast i32 65535 to <32 x i1>
|
|
%mask1 = icmp sgt <32 x i16> %c, %d
|
|
%mask = or <32 x i1> %mask0, %mask1
|
|
%res = select <32 x i1> %mask, <32 x i16> %a, <32 x i16> %b
|
|
ret <32 x i16> %res
|
|
}
|
|
|
|
; Test case 6: v64i1 mask via bitconvert combined with dynamic condition.
|
|
; Verifies the KSET1D submask pattern survives past SelectionDAG combines.
|
|
define <64 x i8> @mask_v64i1_lower32(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
|
|
; AVX512F-LABEL: mask_v64i1_lower32:
|
|
; AVX512F: # %bb.0:
|
|
; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3
|
|
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm2
|
|
; AVX512F-NEXT: vpcmpgtb %ymm3, %ymm2, %ymm2
|
|
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = -1
|
|
; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
|
|
; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))
|
|
; AVX512F-NEXT: retq
|
|
;
|
|
; AVX512DQ-LABEL: mask_v64i1_lower32:
|
|
; AVX512DQ: # %bb.0:
|
|
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm3
|
|
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm2
|
|
; AVX512DQ-NEXT: vpcmpgtb %ymm3, %ymm2, %ymm2
|
|
; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm3 = -1
|
|
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
|
|
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))
|
|
; AVX512DQ-NEXT: retq
|
|
;
|
|
; AVX512BW-LABEL: mask_v64i1_lower32:
|
|
; AVX512BW: # %bb.0:
|
|
; AVX512BW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF
|
|
; AVX512BW-NEXT: kmovq %rax, %k0
|
|
; AVX512BW-NEXT: vpcmpgtb %zmm3, %zmm2, %k1
|
|
; AVX512BW-NEXT: korq %k0, %k1, %k1
|
|
; AVX512BW-NEXT: vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
|
|
; AVX512BW-NEXT: retq
|
|
;
|
|
; AVX512DQBW-LABEL: mask_v64i1_lower32:
|
|
; AVX512DQBW: # %bb.0:
|
|
; AVX512DQBW-NEXT: kxnord %k0, %k0, %k0
|
|
; AVX512DQBW-NEXT: vpcmpgtb %zmm3, %zmm2, %k1
|
|
; AVX512DQBW-NEXT: korq %k0, %k1, %k1
|
|
; AVX512DQBW-NEXT: vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
|
|
; AVX512DQBW-NEXT: retq
|
|
%mask0 = bitcast i64 4294967295 to <64 x i1>
|
|
%mask1 = icmp sgt <64 x i8> %c, %d
|
|
%mask = or <64 x i1> %mask0, %mask1
|
|
%res = select <64 x i1> %mask, <64 x i8> %a, <64 x i8> %b
|
|
ret <64 x i8> %res
|
|
}
|
|
|