derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
608 lines
23 KiB
LLVM
608 lines
23 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512
|
|
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512
|
|
|
|
define void @concat_a_to_shuf_of_a(ptr %a.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_a_to_shuf_of_a:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movdqa (%rdi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
|
|
; SSE-NEXT: movdqa %xmm1, (%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_a_to_shuf_of_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
|
|
; AVX1-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX1-NEXT: vzeroupper
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_a_to_shuf_of_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1]
|
|
; AVX2-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX2-NEXT: vzeroupper
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_a_to_shuf_of_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1]
|
|
; AVX512-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX512-NEXT: vzeroupper
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_shuf_of_a_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_shuf_of_a_to_a:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movdqa (%rdi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, (%rdx)
|
|
; SSE-NEXT: movdqa %xmm1, 16(%rdx)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_a_to_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
|
|
; AVX1-NEXT: vmovaps %ymm0, (%rdx)
|
|
; AVX1-NEXT: vzeroupper
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_a_to_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0]
|
|
; AVX2-NEXT: vmovaps %ymm0, (%rdx)
|
|
; AVX2-NEXT: vzeroupper
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_a_to_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0]
|
|
; AVX512-NEXT: vmovaps %ymm0, (%rdx)
|
|
; AVX512-NEXT: vzeroupper
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_a_to_shuf_of_a_extrause_of_shuf(ptr %a.ptr, ptr %dst, ptr %shuf.escape.ptr) {
|
|
; SSE-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movdqa (%rdi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm1, (%rdx)
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
|
|
; SSE-NEXT: movdqa %xmm1, (%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX1-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm0, 16(%rsi)
|
|
; AVX1-NEXT: vmovaps %xmm1, (%rsi)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX2-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm0, 16(%rsi)
|
|
; AVX2-NEXT: vmovaps %xmm1, (%rsi)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX512-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm0, 16(%rsi)
|
|
; AVX512-NEXT: vmovaps %xmm1, (%rsi)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
store <2 x i64> %shuffle, ptr %shuf.escape.ptr, align 64
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_a_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE2-LABEL: concat_a_to_shuf_of_ab:
|
|
; SSE2: # %bb.0:
|
|
; SSE2-NEXT: movapd (%rdi), %xmm0
|
|
; SSE2-NEXT: movapd (%rsi), %xmm1
|
|
; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
|
|
; SSE2-NEXT: movapd %xmm0, 16(%rdx)
|
|
; SSE2-NEXT: movapd %xmm1, (%rdx)
|
|
; SSE2-NEXT: retq
|
|
;
|
|
; SSE42-LABEL: concat_a_to_shuf_of_ab:
|
|
; SSE42: # %bb.0:
|
|
; SSE42-NEXT: movaps (%rdi), %xmm0
|
|
; SSE42-NEXT: movaps (%rsi), %xmm1
|
|
; SSE42-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
|
|
; SSE42-NEXT: movaps %xmm0, 16(%rdx)
|
|
; SSE42-NEXT: movaps %xmm1, (%rdx)
|
|
; SSE42-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_a_to_shuf_of_ab:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX1-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX1-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_a_to_shuf_of_ab:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_a_to_shuf_of_ab:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX512-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_b_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE2-LABEL: concat_b_to_shuf_of_ab:
|
|
; SSE2: # %bb.0:
|
|
; SSE2-NEXT: movaps (%rsi), %xmm0
|
|
; SSE2-NEXT: movaps %xmm0, %xmm1
|
|
; SSE2-NEXT: movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
|
|
; SSE2-NEXT: movaps %xmm0, 16(%rdx)
|
|
; SSE2-NEXT: movaps %xmm1, (%rdx)
|
|
; SSE2-NEXT: retq
|
|
;
|
|
; SSE42-LABEL: concat_b_to_shuf_of_ab:
|
|
; SSE42: # %bb.0:
|
|
; SSE42-NEXT: movaps (%rsi), %xmm0
|
|
; SSE42-NEXT: movaps (%rdi), %xmm1
|
|
; SSE42-NEXT: blendps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]
|
|
; SSE42-NEXT: movaps %xmm0, 16(%rdx)
|
|
; SSE42-NEXT: movaps %xmm1, (%rdx)
|
|
; SSE42-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_b_to_shuf_of_ab:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX1-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX1-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_b_to_shuf_of_ab:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX2-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_b_to_shuf_of_ab:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX512-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX512-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_shuf_of_ab_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE2-LABEL: concat_shuf_of_ab_to_a:
|
|
; SSE2: # %bb.0:
|
|
; SSE2-NEXT: movapd (%rdi), %xmm0
|
|
; SSE2-NEXT: movapd (%rsi), %xmm1
|
|
; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
|
|
; SSE2-NEXT: movapd %xmm0, (%rdx)
|
|
; SSE2-NEXT: movapd %xmm1, 16(%rdx)
|
|
; SSE2-NEXT: retq
|
|
;
|
|
; SSE42-LABEL: concat_shuf_of_ab_to_a:
|
|
; SSE42: # %bb.0:
|
|
; SSE42-NEXT: movaps (%rdi), %xmm0
|
|
; SSE42-NEXT: movaps (%rsi), %xmm1
|
|
; SSE42-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
|
|
; SSE42-NEXT: movaps %xmm1, 16(%rdx)
|
|
; SSE42-NEXT: movaps %xmm0, (%rdx)
|
|
; SSE42-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_ab_to_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX1-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX1-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_ab_to_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_ab_to_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]
|
|
; AVX512-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
|
%concat = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_shuf_of_ab_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE2-LABEL: concat_shuf_of_ab_to_b:
|
|
; SSE2: # %bb.0:
|
|
; SSE2-NEXT: movaps (%rsi), %xmm0
|
|
; SSE2-NEXT: movaps %xmm0, %xmm1
|
|
; SSE2-NEXT: movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
|
|
; SSE2-NEXT: movaps %xmm1, 16(%rdx)
|
|
; SSE2-NEXT: movaps %xmm0, (%rdx)
|
|
; SSE2-NEXT: retq
|
|
;
|
|
; SSE42-LABEL: concat_shuf_of_ab_to_b:
|
|
; SSE42: # %bb.0:
|
|
; SSE42-NEXT: movaps (%rsi), %xmm0
|
|
; SSE42-NEXT: movaps (%rdi), %xmm1
|
|
; SSE42-NEXT: blendps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]
|
|
; SSE42-NEXT: movaps %xmm1, 16(%rdx)
|
|
; SSE42-NEXT: movaps %xmm0, (%rdx)
|
|
; SSE42-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_ab_to_b:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX1-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX1-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_ab_to_b:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX2-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_ab_to_b:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX512-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
|
|
; AVX512-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>
|
|
%concat = shufflevector <2 x i64> %b, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_b_to_shuf_of_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_b_to_shuf_of_a:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movaps (%rsi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; SSE-NEXT: movaps %xmm0, 16(%rdx)
|
|
; SSE-NEXT: movdqa %xmm1, (%rdx)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_b_to_shuf_of_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX1-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_b_to_shuf_of_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX2-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_b_to_shuf_of_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX512-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX512-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm1, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_shuf_of_a_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_shuf_of_a_to_b:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movaps (%rsi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm1, 16(%rdx)
|
|
; SSE-NEXT: movaps %xmm0, (%rdx)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_a_to_b:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX1-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX1-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_a_to_b:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX2-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX2-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_a_to_b:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rsi), %xmm0
|
|
; AVX512-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]
|
|
; AVX512-NEXT: vmovaps %xmm1, 16(%rdx)
|
|
; AVX512-NEXT: vmovaps %xmm0, (%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %b, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_poison_to_shuf_of_a(ptr %a.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_poison_to_shuf_of_a:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, (%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_poison_to_shuf_of_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX1-NEXT: vmovaps %xmm0, (%rsi)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_poison_to_shuf_of_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX2-NEXT: vmovaps %xmm0, (%rsi)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_poison_to_shuf_of_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX512-NEXT: vmovaps %xmm0, (%rsi)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_shuf_of_a_to_poison(ptr %a.ptr, ptr %b.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_shuf_of_a_to_poison:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rdx)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_a_to_poison:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX1-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_a_to_poison:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_a_to_poison:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; AVX512-NEXT: vmovaps %xmm0, 16(%rdx)
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%b = load <2 x i64>, ptr %b.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> poison, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_shuf_of_a_to_itself(ptr %a.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_shuf_of_a_to_itself:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
|
|
; SSE-NEXT: movdqa %xmm0, (%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_a_to_itself:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vpermilpd {{.*#+}} xmm0 = mem[1,0]
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
|
; AVX1-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX1-NEXT: vzeroupper
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_a_to_itself:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0]
|
|
; AVX2-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX2-NEXT: vzeroupper
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_a_to_itself:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vmovaps (%rdi), %xmm0
|
|
; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0]
|
|
; AVX512-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX512-NEXT: vzeroupper
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat = shufflevector <2 x i64> %shuffle, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
store <4 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
|
|
define void @concat_aaa_to_shuf_of_a(ptr %a.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_aaa_to_shuf_of_a:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movdqa (%rdi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, 32(%rsi)
|
|
; SSE-NEXT: movdqa %xmm0, 48(%rsi)
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
|
|
; SSE-NEXT: movdqa %xmm1, (%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_aaa_to_shuf_of_a:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1
|
|
; AVX1-NEXT: vmovaps %ymm0, 32(%rsi)
|
|
; AVX1-NEXT: vmovaps %ymm1, (%rsi)
|
|
; AVX1-NEXT: vzeroupper
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_aaa_to_shuf_of_a:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,1]
|
|
; AVX2-NEXT: vmovaps %ymm0, 32(%rsi)
|
|
; AVX2-NEXT: vmovaps %ymm1, (%rsi)
|
|
; AVX2-NEXT: vzeroupper
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_aaa_to_shuf_of_a:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1
|
|
; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
|
|
; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
|
|
; AVX512-NEXT: vzeroupper
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat01 = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
%concat23 = shufflevector <2 x i64> %a, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
%concat = shufflevector <4 x i64> %concat01, <4 x i64> %concat23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
store <8 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|
|
define void @concat_shuf_of_a_to_aaa(ptr %a.ptr, ptr %dst) {
|
|
; SSE-LABEL: concat_shuf_of_a_to_aaa:
|
|
; SSE: # %bb.0:
|
|
; SSE-NEXT: movdqa (%rdi), %xmm0
|
|
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; SSE-NEXT: movdqa %xmm0, 32(%rsi)
|
|
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
|
|
; SSE-NEXT: movdqa %xmm0, (%rsi)
|
|
; SSE-NEXT: movdqa %xmm1, 48(%rsi)
|
|
; SSE-NEXT: retq
|
|
;
|
|
; AVX1-LABEL: concat_shuf_of_a_to_aaa:
|
|
; AVX1: # %bb.0:
|
|
; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
|
|
; AVX1-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX1-NEXT: vmovaps %ymm1, 32(%rsi)
|
|
; AVX1-NEXT: vzeroupper
|
|
; AVX1-NEXT: retq
|
|
;
|
|
; AVX2-LABEL: concat_shuf_of_a_to_aaa:
|
|
; AVX2: # %bb.0:
|
|
; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,1,1,0]
|
|
; AVX2-NEXT: vmovaps %ymm0, (%rsi)
|
|
; AVX2-NEXT: vmovaps %ymm1, 32(%rsi)
|
|
; AVX2-NEXT: vzeroupper
|
|
; AVX2-NEXT: retq
|
|
;
|
|
; AVX512-LABEL: concat_shuf_of_a_to_aaa:
|
|
; AVX512: # %bb.0:
|
|
; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
|
|
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
|
|
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
|
|
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
|
|
; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
|
|
; AVX512-NEXT: vzeroupper
|
|
; AVX512-NEXT: retq
|
|
%a = load <2 x i64>, ptr %a.ptr, align 64
|
|
%shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
|
|
%concat01 = shufflevector <2 x i64> %a, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
%concat23 = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
|
|
%concat = shufflevector <4 x i64> %concat01, <4 x i64> %concat23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
store <8 x i64> %concat, ptr %dst, align 64
|
|
ret void
|
|
}
|